fix duplicate geopackage

.github/workflows/build_test_push_docker.yaml

@@ -4,13 +4,21 @@ on:
   push:
     branches:
       - main
+    paths:
+      - 'docker/**'
+      - '.github/workflows/build_test_push_docker.yml'
 
   pull_request:
     branches:
       - main     
+    paths:
+      - 'docker/**'
+      - '.github/workflows/build_test_push_docker.yml'
 
   workflow_run:
-    workflows: ["Test Forcing Processor","Test Datastream Python"]
+    workflows: 
+      - "Test Forcing Processor"
+      - "Test Datastream Python"
     types:
       - completed
 

python_tools/src/python_tools/run_validator.py

@@ -95,17 +95,16 @@ def validate_data_dir(data_dir):
     for path, _, files in os.walk(data_dir):
         for jfile in files:
             jfile_path = os.path.join(path,jfile)
-            if jfile_path.find('config') >= 0:
-                if jfile_path.find('realization') >= 0: 
-                    if realization_file is None: 
-                        realization_file = jfile_path
-                    else: 
-                        raise Exception('This run directory contains more than a single realization file, remove all but one.')
-                if jfile_path.find('.gpkg') >= 0: 
-                    if geopackage_file is None: 
-                        geopackage_file = jfile_path
-                    else: 
-                        raise Exception('This run directory contains more than a single geopackage file, remove all but one.')                    
+            if jfile_path.find('realization') >= 0: 
+                if realization_file is None: 
+                    realization_file = jfile_path
+                else: 
+                    raise Exception('This run directory contains more than a single realization file, remove all but one.')
+            if jfile_path.find('.gpkg') >= 0: 
+                if geopackage_file is None: 
+                    geopackage_file = jfile_path
+                else: 
+                    raise Exception('This run directory contains more than a single geopackage file, remove all but one.')                    
 
     if realization_file is None: 
         raise Exception(f"Did not find realization file in ngen-run/config!!!")

python_tools/tests/test_validator.py

@@ -26,6 +26,16 @@ def test_missing_geopackage():
     except Exception as inst:
         assert inst.__str__() == "Did not find geopackage file in ngen-run/config!!!"
 
+def test_duplicate_geopackage():
+    geo_file = str(TEST_DATA_DIR) + '/config/*.gpkg'
+    geo_file2 = str(TEST_DATA_DIR) + '/config/extra.gpkg'
+    os.system(f"cp {geo_file} {geo_file2}")
+    try:
+        validate_data_dir(TEST_DATA_DIR)
+        assert False 
+    except Exception as inst:
+        assert inst.__str__() == "This run directory contains more than a single geopackage file, remove all but one."        
+
 def test_missing_realization():
     del_file = str(TEST_DATA_DIR) + '/config/*realization*.json'
     os.system(f"rm {del_file}")
@@ -35,6 +45,17 @@ def test_missing_realization():
     except Exception as inst:
         assert inst.__str__() == "Did not find realization file in ngen-run/config!!!"
 
+def test_duplicate_realization():
+    real_file = str(TEST_DATA_DIR) + '/config/*realization*.json'
+    real_file2 = str(TEST_DATA_DIR) + '/config/extra_realization.json'
+    os.system(f"cp {real_file} {real_file2}")
+    try:
+        validate_data_dir(TEST_DATA_DIR)
+        assert False 
+    except Exception as inst:
+        assert inst.__str__() == "This run directory contains more than a single realization file, remove all but one."        
+
+
 def test_missing_bmi_config():
     del_file = str(TEST_DATA_DIR) + '/config/cat_config/CFE/CFE_cat-2586011.ini'
     os.system(f"rm {del_file}")

scripts/docker_builds.sh

@@ -23,6 +23,9 @@ DATASTREAM_DOCKER="$DATASTREAM_PATH"/docker
 cd $DATASTREAM_DOCKER
 echo "Building docker from "$DATASTREAM_DOCKER
 DOCKER_DATASTREAM=$DATASTREAM_DOCKER/ngen-datastream
+if [ -d "$DOCKER_DATASTREAM" ]; then
+  rm -rf $DOCKER_DATASTREAM
+fi
 mkdir $DOCKER_DATASTREAM
 cp -r $DATASTREAM_PATH/forcingprocessor $DOCKER_DATASTREAM/forcingprocessor
 cp -r $DATASTREAM_PATH/python_tools $DOCKER_DATASTREAM/python_tools

scripts/stream.sh

@@ -264,6 +264,7 @@ echo "DATASTREAM_START: $START_TIME" > $DATASTREAM_PROFILING
 NGENRUN_CONFIG="${NGEN_RUN%/}/config"
 NGENRUN_FORCINGS="${NGEN_RUN%/}/forcings"
 NGENRUN_OUTPUT="${NGEN_RUN%/}/outputs"
+NGENRUN_METADATA="${NGEN_RUN%/}/metadata"
 NGENRUN_OUTPUT_NGEN="${NGEN_RUN%/}/outputs/ngen"
 NGENRUN_OUTPUT_PARQUET="${NGEN_RUN%/}/outputs/parquet"
 NGENRUN_OUTPUT_TROUTE="${NGEN_RUN%/}/outputs/troute"
@@ -530,6 +531,7 @@ else
             mkdir -p $DATASTREAM_RESOURCES_NGENFORCINGS
         fi
         cp $NGEN_RUN/forcings/*forcings* $DATASTREAM_RESOURCES_NGENFORCINGS
+        mv $NGENRUN_METADATA $DATASTREAM_META
     fi
 fi