fix bug

CIROH-UA · Sep 30, 2024 · f2f4b4e · f2f4b4e
1 parent ca52e6a
commit f2f4b4e
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 5 deletions.
diff --git a/.github/workflows/build_test_docker_x86.yaml b/.github/workflows/build_test_docker_x86.yaml
@@ -48,4 +48,4 @@ jobs:
     - name: Test docker containers
       run : |
         curl -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg
-        ./scripts/stream.sh -s 202006200100 -e 202006210000 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -n 4
+        ./scripts/stream.sh -s 202006200100 -e 202006210000 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json
diff --git a/forcingprocessor/src/forcingprocessor/processor.py b/forcingprocessor/src/forcingprocessor/processor.py
@@ -731,7 +731,7 @@ def prep_ngen_data(conf):
             jtype in file_types
         ), f"{jtype} for output_file_type is not accepted! Accepted: {file_types}"
     global storage_type
-    if "s3" in output_path:
+    if "s3://" in output_path:
         storage_type = "s3"
     elif "google" in output_path:
         storage_type = "google"

diff --git a/python_tools/src/python_tools/configure_datastream.py b/python_tools/src/python_tools/configure_datastream.py
@@ -64,7 +64,7 @@ def create_conf_fp(date,nprocs,docker_mount,forcing_split_vpu,retro_or_op,geo_ba
     else:
         filename = "filenamelist.txt"
 
-    if len(forcing_split_vpu) > 0:
+    if len(forcing_split_vpu[0]) > 0:
         template = f"https://lynker-spatial.s3-us-west-2.amazonaws.com/hydrofabric/{hf_version}/nextgen/conus_forcing-weights/vpuid%3D$VPU/part-0.parquet"
         gpkg_file = []
         for jvpu in forcing_split_vpu:
@@ -169,7 +169,14 @@ def create_confs(args):
         end_realization   = end_datetime.strftime('%Y-%m-%d %H:%M:%S')
 
         nwm_conf = create_conf_nwm(start_str, end_str, retro_or_op, runinput, urlbaseinput)
-        fp_conf  = create_conf_fp(date_str, conf['globals']['nprocs'],args.docker_mount,forcing_split_vpu,retro_or_op,geo_base,args.hydrofabric_version,run_type) 
+        fp_conf  = create_conf_fp(date_str, 
+                                  conf['globals']['nprocs'],
+                                  args.docker_mount,
+                                  forcing_split_vpu,
+                                  retro_or_op,
+                                  geo_base,
+                                  args.hydrofabric_version,
+                                  run_type) 
 
     else: 
         run_type='non-daily'
@@ -205,7 +212,14 @@ def create_confs(args):
             fp_conf  = create_conf_fp(start, conf['globals']['nprocs'], args.docker_mount, forcing_split_vpu,retro_or_op,geo_base,args.hydrofabric_version,run_type) 
         else:
             nwm_conf = create_conf_nwm(start,end, retro_or_op,runinput,urlbaseinput)
-            fp_conf  = create_conf_fp(start, conf['globals']['nprocs'], args.docker_mount, forcing_split_vpu,retro_or_op,geo_base,args.hydrofabric_version,run_type) 
+            fp_conf  = create_conf_fp(start,
+                                      conf['globals']['nprocs'],
+                                      args.docker_mount, 
+                                      forcing_split_vpu,
+                                      retro_or_op,
+                                      geo_base,
+                                      args.hydrofabric_version,
+                                      run_type) 
 
     conf['nwmurl'] = nwm_conf 
     conf['forcingprocessor'] = nwm_conf    

diff --git a/python_tools/tests/test_configurer.py b/python_tools/tests/test_configurer.py
@@ -86,6 +86,10 @@ def test_conf_basic():
     assert os.path.exists(REALIZATION_META_DS)   
     assert os.path.exists(REALIZATION_RUN) 
 
+    with open(CONF_FP,'r') as fp:
+        data = json.load(fp)   
+    assert data['storage']['output_path'] == "/mounted_dir/ngen-run"  
+
 def test_conf_daily():
     inputs.start_date = "DAILY"
     inputs.end_date   = ""
@@ -195,5 +199,6 @@ def test_conf_daily_short_range_split_vpu():
     with open(CONF_FP,'r') as fp:
         data = json.load(fp)   
     assert len(data['forcing']['gpkg_file']) == 4
+    assert data['storage']['output_path'].startswith("s3://ciroh-community-ngen-datastream/")