Skip to content

Implement weights parquet #49

Implement weights parquet

Implement weights parquet #49

name: Test Datastream Options
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
test-datastream-options:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Configure AWS
run: |
aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws configure set region us-east-1
- name: Build docker containers
run : |
./scripts/docker_builds.sh -b
- name: Install packages for datastream
run: |
curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset && sudo apt-get install git pip pigz awscli python3.9 -y
- name: Get geopackage from hfsubset
run: |
hfsubset -w medium_range -s nextgen -v 2.1.1 -l divides,flowlines,network,nexus,forcing-weights,flowpath-attributes,model-attributes -o palisade.gpkg -t hl "Gages-09106150"
- name: Base test and NWM_RETRO_V3
run: |
sudo rm -rf $(pwd)/data/datastream_test
./scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
- name: Cache resource directory
run: |
mkdir ./data/cache
cp -r ./data/datastream_test/datastream-resources ./data/cache
cp -r ./data/cache/datastream-resources ./data/cache/datastream-resources-no-forcings
cp -r ./data/cache/datastream-resources ./data/cache/datastream-resources-missing
sudo rm -rf ./data/cache/datastream-resources-no-forcings/ngen-forcings
- name: Resource directory test missing all
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
sudo rm -rf ./data/cache/datastream-resources-missing/ngen-forcings
sudo rm -rf ./data/cache/datastream-resources-missing/config/*
./scripts/stream.sh -r ./data/cache/datastream-resources-missing -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg
- name: Resource directory test
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
./scripts/stream.sh -r ./data/cache/datastream-resources -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test
- name: Forcings sources option test NWM_RETRO_V2
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
./scripts/stream.sh -r ./data/cache/datastream-resources-no-forcings -s 201906200100 -e 201906200200 -C NWM_RETRO_V2 -d $(pwd)/data/datastream_test
- name: Forcings sources option test NWM_OPERATIONAL_V3
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
TODAY=$(env TZ=US/Eastern date +'%Y%m%d')
./scripts/stream.sh -r ./data/cache/datastream-resources-no-forcings -s $TODAY"0100" -e $TODAY"0200" -C NWM_OPERATIONAL_V3 -d $(pwd)/data/datastream_test
- name: Forcings sources option test NOMADS_OPERATIONAL
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
TODAY=$(env TZ=US/Eastern date +'%Y%m%d')
./scripts/stream.sh -r ./data/cache/datastream-resources-no-forcings -s $TODAY"0100" -e $TODAY"0200" -C NOMADS_OPERATIONAL -d $(pwd)/data/datastream_test
- name: Test hfsubset options
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
./scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -I "Gages-09106150" -i hl -v 2.1.1 -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
- name: S3 write out test
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
./scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -S ngen-datastream -o git_actions_test/
aws s3api wait object-exists --bucket ngen-datastream --key git_actions_test/ngen-run.tar.gz
aws s3api delete-object --bucket ngen-datastream --key git_actions_test/ngen-run.tar.gz
- name: DAILY today test
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
./scripts/stream.sh -s DAILY -C NWM_OPERATIONAL_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
- name: DAILY pick day test
if: always()
run: |
sudo rm -rf $(pwd)/data/datastream_test
./scripts/stream.sh -s DAILY -e 202006200000 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json