219 refactor workflow automation to use berkley metadata models

nmdc_automation/__init__.py

@@ -1,4 +1,4 @@
 from .api import nmdcapi
 from .config import config
 from .import_automation import activity_mapper
-from .workflow_automation import watch_nmdc, wfutils, workflows, activities
+from .workflow_automation import watch_nmdc, wfutils, workflows, workflow_process

nmdc_automation/api/nmdcapi.py

@@ -10,26 +10,30 @@
 import mimetypes
 from pathlib import Path
 from time import time
-from typing import Union
+from typing import Union, List
 from datetime import datetime, timedelta, timezone
 from nmdc_automation.config import Config, UserConfig
 import logging
 
 
-def _get_sha256(fn):
-    hashfn = fn + ".sha256"
-    if os.path.exists(hashfn):
-        with open(hashfn) as f:
+def _get_sha256(fn: Union[str, Path]) -> str:
+    """
+    Helper function to get the sha256 hash of a file if it exists.
+    """
+    shahash = hashlib.sha256()
+    if isinstance(fn, str):
+        fn = Path(fn)
+    hash_fn = fn.with_suffix(".sha256")
+    if hash_fn.exists():
+        with hash_fn.open() as f:
             sha = f.read().rstrip()
     else:
-        logging.info("hashing %s" % (fn))
-        shahash = hashlib.sha256()
-        with open(fn, "rb") as f:
-            # Read and update hash string value in blocks of 4K
+        logging.info(f"hashing {fn}")
+        with fn.open("rb") as f:
             for byte_block in iter(lambda: f.read(1048576), b""):
                 shahash.update(byte_block)
         sha = shahash.hexdigest()
-        with open(hashfn, "w") as f:
+        with hash_fn.open("w") as f:
             f.write(sha)
             f.write("\n")
     return sha
@@ -46,8 +50,10 @@ class NmdcRuntimeApi:
     client_id = None
     client_secret = None
 
-    def __init__(self, site_configuration: Union[str, Path]):
-        self.config = Config(site_configuration)
+    def __init__(self, site_configuration: Union[str, Path, Config]):
+        if isinstance(site_configuration, str) or isinstance(site_configuration, Path):
+            site_configuration = Config(site_configuration)
+        self.config = site_configuration
         self._base_url = self.config.api_url
         self.client_id = self.config.client_id
         self.client_secret = self.config.client_secret
@@ -184,7 +190,7 @@ def create_object(self, fn, description, dataurl):
 
     @refresh_token
     def post_objects(self, obj_data):
-        url = self._base_url + "workflows/activities"
+        url = self._base_url + "workflows/workflow_executions"
 
         resp = requests.post(url, headers=self.header, data=json.dumps(obj_data))
         return resp.json()
@@ -206,7 +212,7 @@ def bump_time(self, obj):
         return resp.json()
 
     @refresh_token
-    def list_jobs(self, filt=None, max=100):
+    def list_jobs(self, filt=None, max=100) -> List[dict]:
         url = "%sjobs?max_page_size=%s" % (self._base_url, max)
         d = {}
         if filt:
@@ -316,6 +322,7 @@ def run_query(self, query):
         return resp.json()
 
 
+# TODO - This is deprecated and should be removed along with the re_iding code that uses it
 class NmdcRuntimeUserApi:
     """
     Basic Runtime API Client with user/password authentication.

nmdc_automation/config/workflows/workflows-mt.yaml

@@ -1,27 +1,27 @@
 Workflows:
   - Name: Sequencing Noninterleaved
-    Collection: omics_processing_set
+    Collection: data_generation_set
     Enabled: True
     Analyte Category: Metatranscriptome
     Filter Output Objects:
     - Metagenome Raw Read 1
     - Metagenome Raw Read 2
 
   - Name: Sequencing Interleaved
-    Collection: omics_processing_set
+    Collection: data_generation_set
     Enabled: True
     Analyte Category: Metatranscriptome
     Filter Output Objects:
     - Metagenome Raw Reads
 
   - Name: Metatranscriptome Reads QC
-    Type: nmdc:ReadQcAnalysisActivity
+    Type: nmdc:ReadQcAnalysis
     Enabled: True
     Analyte Category: Metatranscriptome
     Git_repo: https://github.com/microbiomedata/metaT_ReadsQC
     Version: v0.0.7
     WDL: rqcfilter.wdl
-    Collection: read_qc_analysis_activity_set
+    Collection: workflow_execution_set
     Filter Input Objects:
     - Metagenome Raw Reads
     Predecessors:
@@ -30,14 +30,14 @@ Workflows:
     Input_prefix: nmdc_rqcfilter
     Inputs:
       input_files: do:Metagenome Raw Reads
-      proj: "{activity_id}"
-    Activity:
-      name: "Read QC Activity for {id}"
+      proj: "{workflow_execution_id}"
+    Workflow Execution:
+      name: "Read QC for {id}"
       input_read_bases: "{outputs.stats.input_read_bases}"
       input_read_count: "{outputs.stats.input_read_count}"
       output_read_bases: "{outputs.stats.output_read_bases}"
       output_read_count: "{outputs.stats.output_read_count}"
-      type: nmdc:ReadQcAnalysisActivity
+      type: nmdc:ReadQcAnalysis
     Outputs:
       - output: filtered_final
         name: Reads QC result fastq (clean data)
@@ -57,30 +57,30 @@ Workflows:
         description: "rRNA fastq for {id}"
 
   - Name: Metatranscriptome Reads QC Interleave
-    Type: nmdc:ReadQcAnalysisActivity
+    Type: nmdc:ReadQcAnalysis
     Enabled: True
     Analyte Category: Metatranscriptome
     Git_repo: https://github.com/microbiomedata/metaT_ReadsQC
     Version: v0.0.7
-    Collection: read_qc_analysis_activity_set
+    Collection: workflow_execution_set
     WDL: interleave_rqcfilter.wdl
     Input_prefix: nmdc_rqcfilter
     Inputs:
-      proj: "{activity_id}"
+      proj: "{workflow_execution_id}"
       input_fastq1: do:Metagenome Raw Read 1
       input_fastq2: do:Metagenome Raw Read 2
     Filter Input Objects:
     - Metagenome Raw Read 1
     - Metagenome Raw Read 2
     Predecessors:
     - Sequencing Noninterleaved
-    Activity:
-      name: "Read QC Activity for {id}"
+    Workflow Execution:
+      name: "Read QC for {id}"
       input_read_bases: "{outputs.stats.input_read_bases}"
       input_read_count: "{outputs.stats.input_read_count}"
       output_read_bases: "{outputs.stats.output_read_bases}"
       output_read_count: "{outputs.stats.output_read_count}"
-      type: nmdc:ReadQcAnalysisActivity
+      type: nmdc:ReadQcAnalysis
     Outputs:
       - output: filtered_final
         name: Reads QC result fastq (clean data)
@@ -106,16 +106,16 @@ Workflows:
     Git_repo: https://github.com/microbiomedata/metaT_Assembly
     Version: v0.0.2
     WDL: metaT_assembly.wdl
-    Collection: metatranscriptome_assembly_set
+    Collection: workflow_execution_set
     Predecessors:
     - Metatranscriptome Reads QC
     - Metatranscriptome Reads QC Interleave
     Input_prefix: jgi_metaASM
     Inputs:
       input_files: do:Filtered Sequencing Reads
-      proj: "{activity_id}"
-    Activity:
-      name: "Metatranscriptome Assembly Activity for {id}"
+      proj: "{workflow_execution_id}"
+    Workflow Execution:
+      name: "Metatranscriptome Assembly for {id}"
       type: nmdc:MetatranscriptomeAssembly
       asm_score: "{outputs.stats.asm_score}"
       contig_bp: "{outputs.stats.contig_bp}"
@@ -153,23 +153,23 @@ Workflows:
         description: "Alignment index file for {id}" 
 
   - Name: Metatranscriptome Annotation
-    Type: nmdc:MetatranscriptomeAnnotationActivity
+    Type: nmdc:MetatranscriptomeAnnotation
     Enabled: True
     Analyte Category: Metatranscriptome
     Git_repo: https://github.com/microbiomedata/mg_annotation
     Version: v1.1.4
     WDL: annotation_full.wdl
-    Collection: metatranscriptome_annotation_set
+    Collection: workflow_execution_set
     Predecessors:
     - Metatranscriptome Assembly
     Input_prefix: annotation
     Inputs:
       input_file: do:Assembly Contigs
       imgap_project_id: "scaffold"
-      proj: "{activity_id}"
-    Activity:
-      name: "Metatranscriptome Annotation Analysis Activity for {id}"
-      type: nmdc:MetatranscriptomeAnnotationActivity
+      proj: "{workflow_execution_id}"
+    Workflow Execution:
+      name: "Metatranscriptome Annotation Analysis for {id}"
+      type: nmdc:MetatranscriptomeAnnotation
     Outputs:
       - output: proteins_faa
         data_object_type: Annotation Amino Acid FASTA
@@ -282,7 +282,7 @@ Workflows:
     Git_repo: https://github.com/microbiomedata/metaT_ReadCounts
     Version: v0.0.5
     WDL: readcount.wdl
-    Collection: metatranscriptome_expression_analysis_set
+    Collection: workflow_execution_set
     Predecessors:
     - Metatranscriptome Annotation
     Input_prefix: nmdc_expression
@@ -291,8 +291,8 @@ Workflows:
       map: do:Contig Mapping File
       bam: do:Assembly Coverage BAM
       rna_type: "aRNA"
-      proj: "{activity_id}"
-    Activity:
+      proj: "{workflow_execution_id}"
+    Workflow Execution:
       name: "Metatranscriptome Expression Analysis for {id}"
       type: nmdc:MetatranscriptomeExpressionAnalysis
     Outputs:
@@ -312,16 +312,16 @@ Workflows:
     Git_repo: https://github.com/microbiomedata/metaT_ReadCounts
     Version: v0.0.5
     WDL: readcount.wdl
-    Collection: metatranscriptome_expression_analysis_set
+    Collection: workflow_execution_set
     Predecessors:
     - Metatranscriptome Annotation
     Input_prefix: nmdc_expression
     Inputs:
       gff_file: do:Functional Annotation GFF
       map: do:Contig Mapping File
       bam: do:Assembly Coverage BAM
-      proj: "{activity_id}"
-    Activity:
+      proj: "{workflow_execution_id}"
+    Workflow Execution:
       name: "Metatranscriptome Expression Analysis for {id}"
       type: nmdc:MetatranscriptomeExpressionAnalysis
     Outputs:
@@ -341,7 +341,7 @@ Workflows:
     Git_repo: https://github.com/microbiomedata/metaT_ReadCounts
     Version: v0.0.5
     WDL: readcount.wdl
-    Collection: metatranscriptome_expression_analysis_set
+    Collection: workflow_execution_set
     Predecessors:
     - Metatranscriptome Annotation
     Input_prefix: nmdc_expression
@@ -350,8 +350,8 @@ Workflows:
       map: do:Contig Mapping File
       bam: do:Assembly Coverage BAM
       rna_type: "non_stranded_RNA"
-      proj: "{activity_id}"
-    Activity:
+      proj: "{workflow_execution_id}"
+    Workflow Execution:
       name: "Metatranscriptome Expression Analysis for {id}"
       type: nmdc:MetatranscriptomeExpressionAnalysis
     Outputs: