Merge pull request #121 from Sage-Bionetworks/jbeck/AG-1144/transform…

…_gene_info_testing

Jbeck/ag 1144/transform gene info testing

src/agoradatatools/etl/transform/gene_info.py

@@ -5,8 +5,8 @@
 
 
 def transform_gene_info(
-    datasets: dict, adjusted_p_value_threshold, protein_level_threshold
-):
+    datasets: dict, adjusted_p_value_threshold: float, protein_level_threshold: float
+) -> pd.DataFrame:
     """
     This function will perform transformations and incrementally create a dataset called gene_info.
     Each dataset will be left_joined onto gene_info, starting with gene_metadata.
@@ -81,6 +81,7 @@ def transform_gene_info(
         drop_columns=["ensembl_gene_id"],
     )
 
+    biodomains = biodomains.dropna(subset=["biodomain", "ensembl_gene_id"])
     biodomains = (
         biodomains.groupby("ensembl_gene_id")["biodomain"]
         .apply(set)  # ensure unique biodomain names
@@ -92,34 +93,39 @@ def transform_gene_info(
     # sort biodomains list alphabetically
     biodomains["biodomains"] = biodomains["biodomains"].apply(sorted)
 
+    # Type-check the 'is_adi' and 'is_tep' columns of tep_info to make sure they are booleans and not strings.
+    # Explicitly make NaN is_adi and is_tep values "False" to avoid having to check for boolean and NaN in the
+    # check below.
+    tep_info = tep_info.fillna({"is_adi": False, "is_tep": False})
+    if tep_info["is_adi"].dtype != bool:
+        raise TypeError(
+            f"'is_adi' column must be 'bool', current type is {tep_info['is_adi'].dtype}"
+        )
+    if tep_info["is_tep"].dtype != bool:
+        raise TypeError(
+            f"'is_tep' column must be 'bool', current type is {tep_info['is_tep'].dtype}"
+        )
+
     # For genes with either is_adi or is_tep set to True, create a resource URL that opens
     # the portal page to the specific gene. This must be done using the hgnc_symbol from the
     # tep_info file and not the symbol in gene_info, because there are some mismatches
     # between the two and the hgnc_symbol from tep_info is the correct one to use here.
     # resource_url should be NA if both is_adi and is_tep are false.
-    resource_url_prefix = "https://adknowledgeportal.synapse.org/Explore/Target%20Enabling%20Resources?QueryWrapper0=%7B%22sql%22%3A%22select%20*%20from%20syn26146692%20WHERE%20%60isPublic%60%20%3D%20true%22%2C%22limit%22%3A25%2C%22offset%22%3A0%2C%22selectedFacets%22%3A%5B%7B%22concreteType%22%3A%22org.sagebionetworks.repo.model.table.FacetColumnValuesRequest%22%2C%22columnName%22%3A%22target%22%2C%22facetValues%22%3A%5B%22"
-    resource_url_suffix = "%22%5D%7D%5D%7D"
-    tep_info["resource_url"] = tep_info.apply(
-        lambda row: resource_url_prefix + row["hgnc_symbol"] + resource_url_suffix
-        if row["is_adi"] or row["is_tep"]
-        else np.NaN,
-        axis=1,
+    RESOURCE_URL_PREFIX = (
+        "https://adknowledgeportal.synapse.org/Explore/Target%20Enabling%20Resources?QueryWrapper0=%7B%22sql%22%3A%22"
+        + "select%20*%20from%20syn26146692%20WHERE%20%60isPublic%60%20%3D%20true%22%2C%22limit%22%3A25%2C%22offset%22"
+        + "%3A0%2C%22selectedFacets%22%3A%5B%7B%22concreteType%22%3A%22org.sagebionetworks.repo.model.table."
+        + "FacetColumnValuesRequest%22%2C%22columnName%22%3A%22target%22%2C%22facetValues%22%3A%5B%22"
     )
+    RESOURCE_URL_SUFFIX = "%22%5D%7D%5D%7D"
 
-    ensembl_info = gene_metadata[
-        [
-            "ensembl_gene_id",
-            "ensembl_release",
-            "ensembl_possible_replacements",
-            "ensembl_permalink",
-        ]
-    ]
-    ensembl_info = nest_fields(
-        df=ensembl_info,
-        grouping="ensembl_gene_id",
-        new_column="ensembl_info",
-        drop_columns=["ensembl_gene_id"],
-        nested_field_is_list=False,
+    tep_info["resource_url"] = tep_info.apply(
+        lambda row: (
+            RESOURCE_URL_PREFIX + row["hgnc_symbol"] + RESOURCE_URL_SUFFIX
+            if row["is_adi"] is True or row["is_tep"] is True
+            else np.NaN
+        ),
+        axis=1,
     )
 
     # Merge all the datasets
@@ -135,7 +141,6 @@ def transform_gene_info(
         druggability,
         biodomains,
         tep_info,
-        ensembl_info,
     ]:
         gene_info = pd.merge(
             left=gene_info,
@@ -159,12 +164,41 @@ def transform_gene_info(
         inplace=True,
     )
 
-    # fillna doesn't work for creating an empty array, need this function instead
-    gene_info["alias"] = gene_info.apply(
-        lambda row: row["alias"]
-        if isinstance(row["alias"], np.ndarray)
-        else np.ndarray(0, dtype=object),
-        axis=1,
+    # fillna doesn't work for creating an empty array, need this function instead for alias and possible replacements
+    gene_info["alias"] = gene_info["alias"].apply(
+        lambda row: row if isinstance(row, np.ndarray) else np.ndarray(0, dtype=object)
+    )
+
+    gene_info["ensembl_possible_replacements"] = gene_info[
+        "ensembl_possible_replacements"
+    ].apply(
+        lambda row: row if isinstance(row, np.ndarray) else np.ndarray(0, dtype=object)
+    )
+
+    # Add ensembl_info as a nested field. This is done after merging all other data sets so it applies to
+    # all possible Ensembl IDs in all data sets.
+    ensembl_info = gene_info[
+        [
+            "ensembl_gene_id",
+            "ensembl_release",
+            "ensembl_possible_replacements",
+            "ensembl_permalink",
+        ]
+    ]
+    ensembl_info = nest_fields(
+        df=ensembl_info,
+        grouping="ensembl_gene_id",
+        new_column="ensembl_info",
+        drop_columns=["ensembl_gene_id"],
+        nested_field_is_list=False,
+    )
+
+    gene_info = pd.merge(
+        left=gene_info,
+        right=ensembl_info,
+        on="ensembl_gene_id",
+        how="outer",
+        validate="one_to_one",
     )
 
     gene_info["rna_brain_change_studied"] = gene_info["adj_p_val"] != -1
@@ -179,9 +213,11 @@ def transform_gene_info(
 
     # create 'total_nominations' field
     gene_info["total_nominations"] = gene_info.apply(
-        lambda row: len(row["target_nominations"])
-        if isinstance(row["target_nominations"], list)
-        else np.NaN,
+        lambda row: (
+            len(row["target_nominations"])
+            if isinstance(row["target_nominations"], list)
+            else np.NaN
+        ),
         axis=1,
     )
 

src/agoradatatools/etl/transform/genes_biodomains.py

@@ -103,7 +103,7 @@ def transform_genes_biodomains(datasets: dict) -> pd.DataFrame:
         df=genes_biodomains,
         grouping="ensembl_gene_id",
         new_column="gene_biodomains",
-        drop_columns="ensembl_gene_id",
+        drop_columns=["ensembl_gene_id"],
     )
 
     return genes_biodomains

tests/test_assets/gene_info/input/diff_exp_data_good_input.csv

@@ -0,0 +1,16 @@
+model,tissue,comparison,ensembl_gene_id,logfc,ci_l,ci_r,aveexpr,t,p_value,adj_p_val,gene_biotype,chromosome_name,direction,hgnc_symbol,percentage_gc_content,gene_length,sex,study
+Diagnosis,TCX,AD-CONTROL,ENSG00000001626,-0.384,-0.61,-0.157,1.1,-3.32,0.001,0.003,protein_coding,7.0,DOWN,CFTR,36.59,250188,ALL,MAYO
+Diagnosis,TCX,AD-CONTROL,ENSG00000001631,0.096,0.028,0.164,3.66,2.79,0.005,0.013,protein_coding,7.0,NONE,KRIT1,36.48,47198,ALL,MAYO
+,TCX,AD-CONTROL,ENSG00000001629,0.069,-0.034,0.171,6.84,1.32,0.188,0.267,protein_coding,7.0,NONE,ANKIB1,36.83,155410,ALL,MAYO
+Diagnosis,,AD-CONTROL,ENSG00000001460,-0.043,-0.134,0.048,4.53,-0.92,0.357,0.452,protein_coding,1.0,NONE,,44.09,59936,ALL,MAYO
+Diagnosis,TCX,,ENSG00000000419,-0.011,-0.073,0.052,4.98,-0.34,0.737,0.799,protein_coding,20.0,NONE,DPM1,39.85,23689,ALL,MAYO
+Diagnosis,IFG,AD-CONTROL,ENSG00000000419,-0.088,-0.182,0.007,4.32,-1.82,0.069,0.155,,20.0,NONE,DPM1,39.85,23689,ALL,MSSM
+Diagnosis,IFG,,ENSG00000001629,0.08,0.007,0.154,7.93,2.16,0.031,0.085,protein_coding,,NONE,ANKIB1,36.83,155410,ALL,MSSM
+Diagnosis,IFG,AD-CONTROL,ENSG00000001460,-0.034,-0.114,0.046,3.81,-0.84,0.4,0.554,protein_coding,1.0,,STPG1,44.09,59936,,MSSM
+Diagnosis,IFG,AD-CONTROL,ENSG00000001631,-0.029,-0.095,0.037,4.12,-0.86,0.392,0.547,protein_coding,7.0,NONE,KRIT1,36.48,47198,ALL,
+Diagnosis,IFG,AD-CONTROL,ENSG00000001626,0.026,-0.128,0.179,3.01,0.33,0.741,0.834,protein_coding,7.0,NONE,,36.59,250188,ALL,MSSM
+Diagnosis,DLPFC,AD-CONTROL,ENSG00000001626,-0.143,-0.343,0.058,-0.3,-1.39,0.163,0.296,protein_coding,7.0,NONE,CFTR,36.59,250188,ALL,
+Diagnosis,DLPFC,AD-CONTROL,ENSG00000000419,-0.088,-0.131,-0.045,3.93,-4.03,0.0,0.001,protein_coding,20.0,NONE,DPM1,39.85,23689,ALL,ROSMAP
+Diagnosis,DLPFC,AD-CONTROL,ENSG00000001629,0.084,0.035,0.133,6.65,3.39,0.001,0.005,protein_coding,7.0,NONE,ANKIB1,36.83,155410,ALL,ROSMAP
+Diagnosis,DLPFC,AD-CONTROL,ENSG00000001460,-0.04,-0.092,0.012,4.02,-1.51,0.131,0.252,protein_coding,1.0,NONE,STPG1,44.09,59936,ALL,ROSMAP
+Diagnosis,DLPFC,AD-CONTROL,ENSG00000001631,-0.028,-0.07,0.014,2.94,-1.32,0.187,0.327,protein_coding,7.0,NONE,KRIT1,36.48,47198,ALL,ROSMAP

tests/test_assets/gene_info/input/diff_exp_data_type_error.csv

@@ -0,0 +1,3 @@
+model,tissue,comparison,ensembl_gene_id,logfc,ci_l,ci_r,aveexpr,t,p_value,adj_p_val,gene_biotype,chromosome_name,direction,hgnc_symbol,percentage_gc_content,gene_length,sex,study
+Diagnosis,TCX,AD-CONTROL,ENSG00000001626,-0.384,-0.61,-0.157,1.1,-3.32,0.001,string_value,protein_coding,7.0,DOWN,CFTR,36.59,250188,ALL,MAYO
+Diagnosis,TCX,AD-CONTROL,ENSG00000001631,0.096,0.028,0.164,3.66,2.79,0.005,0.013,protein_coding,7.0,NONE,KRIT1,36.48,47198,ALL,MAYO

tests/test_assets/gene_info/input/druggability_good_input.csv

@@ -0,0 +1,6 @@
+ensembl_gene_id,sm_druggability_bucket,safety_bucket,feasibility_bucket,abability_bucket,new_modality_bucket,tissue_engagement_bucket,pharos_class,classification,safety_bucket_definition,feasibility_bucket_definition,abability_bucket_definition,new_modality_bucket_definition,tissue_engagement_bucket_definition
+ENSG00000000005,7,4,4,3,4,4,Tbio,,Safety definition 1,Feasibility definition 1,Abability definition 1,Modaility definition 1,Tissue engagement definition 1
+ENSG00000001036,1,3,5,1,4,2,Tchem,Classification 2,,Feasibility definition 2,Abability definition 2,Modaility definition 2,Tissue engagement definition 2
+ENSG00000000460,13,3,5,3,4,4,Tdark,Classification 3,Safety definition 3,,,Modaility definition 3,Tissue engagement definition 3
+ENSG00000000971,3,4,3,1,4,5,Tbio,Classification 4,Safety definition 4,Feasibility definition 4,Abability definition 4,,Tissue engagement definition 4
+ENSG00000001084,1,5,3,3,4,2,,Classification 5,Safety definition 5,Feasibility definition 5,Abability definition 5,Modaility definition 5,

tests/test_assets/gene_info/input/eqtl_good_input.csv

@@ -0,0 +1,7 @@
+ensembl_gene_id,is_eqtl
+ENSG00000000419,True
+ENSG00000000971,True
+ENSG00000001460,True
+ENSG00000001626,True
+ENSG00000161149,
+ENSG00000001517,False

tests/test_assets/gene_info/input/eqtl_merge_error.csv

@@ -0,0 +1,4 @@
+ensembl_gene_id,is_eqtl
+ENSG00000000419,True
+ENSG00000000971,True
+ENSG00000000419,True

tests/test_assets/gene_info/input/genes_biodomains_good_input.csv

@@ -0,0 +1,7 @@
+biodomain,abbr,label,color,go_id,goterm_name,n_symbol,symbol,ensembl_gene_id
+Synapse,Sy,Synapse [Sy],#329a33,GO:0005102,signaling receptor binding,362.0,TNMD,ENSG00000000005
+Proteostasis,Pr,Proteostasis [Pr],#c8b269,,endopeptidase activity,73.0,TNMD,ENSG00000000005
+Apoptosis,Ap,Apoptosis [Ap],,GO:0006915,apoptotic process,577.0,DPM1,ENSG00000000419
+Structural Stabilization,,,#ff9a9a,GO:0030863,cortical cytoskeleton,21.0,SCYL3,ENSG00000000457
+Synapse,Sy,Synapse [Sy],#329a33,GO:0034704,,,TNMD,ENSG00000000005
+,Sy,Synapse [Sy],#329a33,GO:0034704,,,TNMD,ENSG00000000005

tests/test_assets/gene_info/input/igap_good_input.csv

@@ -0,0 +1,7 @@
+ensembl_gene_id,hgnc_symbol
+ENSG00000000419,ADAMTS1
+ENSG00000000460,APH1B
+ENSG00000000971,
+ENSG00000001084,UNC5CL
+ENSG00000001460,ICA1
+,ICA1

tests/test_assets/gene_info/input/igap_merge_error.csv

@@ -0,0 +1,4 @@
+ensembl_gene_id,hgnc_symbol
+ENSG00000000419,ADAMTS1
+ENSG00000000460,APH1B
+ENSG00000000460,

tests/test_assets/gene_info/input/median_expression_good_input.csv

@@ -0,0 +1,11 @@
+ensembl_gene_id,min,first_quartile,median,mean,third_quartile,max,tissue
+ENSG00000000419,4.14,4.82,4.99,4.98,5.12,5.47,TCX
+ENSG00000000457,1.87,3.31,3.56,3.53,3.79,4.29,TCX
+ENSG00000000971,2.38,3.78,4.78,4.66,5.42,7.48,
+ENSG00000001036,2.55,3.42,3.69,3.67,3.9,4.69,TCX
+ENSG00000001631,2.39,3.43,3.58,3.56,3.71,4.0,TCX
+ENSG00000000419,2.46,3.81,4.11,4.11,4.4,5.85,DLPFC
+ENSG00000000971,1.75,3.69,4.33,4.38,4.99,8.17,DLPFC
+ENSG00000000419,2.43,3.82,4.17,4.13,4.5,5.35,IFG
+ENSG00000001036,2.04,3.44,3.64,3.63,3.86,4.8,IFG
+ENSG00000001631,3.07,3.75,3.88,3.88,4.01,4.42,IFG

tests/test_assets/gene_info/input/proteomics_good_input.csv

@@ -0,0 +1,17 @@
+uniqid,genename,uniprotid,ensembl_gene_id,tissue,log2_fc,ci_upr,ci_lwr,pval,cor_pval
+DPM1|O60762,DPM1,O60762,ENSG00000000419,DLPFC,-0.001,0.109,-0.11,1.0,1.0
+GCLC|P48506,GCLC,P48506,ENSG00000001084,DLPFC,0.172,0.241,0.104,0.0,0.0
+CFH|P08603,CFH,P08603,ENSG00000000971,DLPFC,,,,,
+CYP51A1|Q16850,CYP51A1,Q16850,ENSG00000001630,DLPFC,,,,,
+DPM1|O60762,DPM1,O60762,ENSG00000000419,MFG,0.175,0.814,-0.463,0.784,1.0
+GCLC|P48506,GCLC,P48506,ENSG00000001084,MFG,0.011,0.23,-0.207,0.991,1.0
+CFH|P08603,CFH,P08603,ENSG00000000971,,0.005,0.8,-0.79,1.0,1.0
+CYP51A1|Q16850,CYP51A1,Q16850,ENSG00000001630,MFG,,,,,
+DPM1|O60762,DPM1,,ENSG00000000419,TCX,-0.149,0.011,-0.309,0.073,0.383
+GCLC|P48506,GCLC,P48506,ENSG00000001084,TCX,-0.043,0.042,-0.128,0.462,1.0
+CFH|P08603,CFH,P08603,ENSG00000000971,TCX,,,,,
+CYP51A1|Q16850,CYP51A1,Q16850,ENSG00000001630,TCX,,,,,
+DPM1|O60762,DPM1,O60762,ENSG00000000419,AntPFC,0.052,0.215,-0.111,0.734,1.0
+GCLC|P48506,,P48506,ENSG00000001084,AntPFC,-0.023,0.077,-0.123,0.848,1.0
+CFH|P08603,CFH,P08603,ENSG00000000971,AntPFC,,,,,
+,CYP51A1,Q16850,ENSG00000001630,AntPFC,0.265,0.567,-0.037,0.099,0.565

tests/test_assets/gene_info/input/proteomics_srm_good_input.csv

@@ -0,0 +1,7 @@
+uniqid,genename,uniprotid,ensembl_gene_id,tissue,log2_fc,ci_upr,ci_lwr,pval,cor_pval
+CD2AP|Q9Y5K6,CD2AP,Q9Y5K6,ENSG00000001630,DLPFC,0.026,0.102,-0.05,0.696,0.952
+,SNCA,P37840,ENSG00000001629,DLPFC,-0.026,0.028,-0.08,0.505,0.72
+NDUFA7|O95182,,O95182,ENSG00000000419,DLPFC,-0.083,-0.034,-0.131,0.0,0.001
+DIP2B|Q9P265,DIP2B,,ENSG00000001036,DLPFC,0.002,0.072,-0.068,0.997,
+VSNL1|P62760,VSNL1,P62760,ENSG00000000971,,-0.014,0.015,-0.042,0.493,0.718
+SYT11|Q9BT88,SYT11,Q9BT88,ENSG00000000005,DLPFC,0.029,0.074,-0.016,0.281,0.496

tests/test_assets/gene_info/input/proteomics_srm_type_error.csv

@@ -0,0 +1,3 @@
+uniqid,genename,uniprotid,ensembl_gene_id,tissue,log2_fc,ci_upr,ci_lwr,pval,cor_pval
+CD2AP|Q9Y5K6,CD2AP,Q9Y5K6,ENSG00000001630,DLPFC,0.026,0.102,-0.05,0.696,string_value
+,SNCA,P37840,ENSG00000001629,DLPFC,-0.026,0.028,-0.08,0.505,0.72

tests/test_assets/gene_info/input/proteomics_tmt_good_input.csv

@@ -0,0 +1,7 @@
+uniqid,genename,uniprotid,ensembl_gene_id,tissue,log2_fc,ci_upr,ci_lwr,pval,cor_pval
+CYP51A1|A0A0C4DFL7,CYP51A1,A0A0C4DFL7,ENSG00000001630,DLPFC,-0.097,-0.025,-0.168,0.009,0.06
+ANKIB1|Q9P2G1,ANKIB1,Q9P2G1,ENSG00000001629,,-0.023,-0.003,-0.044,0.027,0.124
+,DPM1,H0Y368,ENSG00000000419,DLPFC,-0.032,-0.002,-0.062,0.038,0.153
+FUCA2|Q9BTY2,,Q9BTY2,ENSG00000001036,DLPFC,-0.089,0.006,-0.184,0.066,0.213
+CFH|P08603,CFH,P08603,ENSG00000000971,DLPFC,,0.156,-0.004,0.062,0.205
+KRIT1|O00522,KRIT1,,ENSG00000001631,DLPFC,0.053,0.121,-0.015,0.128,0.319

tests/test_assets/gene_info/input/proteomics_tmt_type_error.csv

@@ -0,0 +1,3 @@
+uniqid,genename,uniprotid,ensembl_gene_id,tissue,log2_fc,ci_upr,ci_lwr,pval,cor_pval
+CYP51A1|A0A0C4DFL7,CYP51A1,A0A0C4DFL7,ENSG00000001630,DLPFC,-0.097,-0.025,-0.168,0.009,string_value
+ANKIB1|Q9P2G1,ANKIB1,Q9P2G1,ENSG00000001629,,-0.023,-0.003,-0.044,0.027,0.124

tests/test_assets/gene_info/input/proteomics_type_error.csv

@@ -0,0 +1,3 @@
+uniqid,genename,uniprotid,ensembl_gene_id,tissue,log2_fc,ci_upr,ci_lwr,pval,cor_pval
+DPM1|O60762,DPM1,O60762,ENSG00000000419,DLPFC,-0.001,0.109,-0.11,1.0,string_value
+GCLC|P48506,GCLC,P48506,ENSG00000001084,DLPFC,0.172,0.241,0.104,0.0,0.0

tests/test_assets/gene_info/input/target_list_good_input.csv

@@ -0,0 +1,6 @@
+source,team,rank,ensembl_gene_id,hgnc_symbol,target_choice_justification,predicted_therapeutic_direction,data_used_to_support_target_selection,data_synapseid,study,input_data,validation_study_details,initial_nomination
+Source_1,Team_1,17-100,ENSG00000000005,TNMD,Justification 1,Prediction 1,Support 1,syn12345,Study_1,"Genetics, RNA, Protein, Clinical",,2018.0
+Source_1,Team_2,,ENSG00000000419,DPM1,Justification 2,Prediction 2,Support 2,,Study_1,,Validation 2,2023.0
+Source_2,Team_3,12,ENSG00000000419,DPM1,Justification 3,,Support 3,,Study_2,RNA,Validation 3,2022.0
+Source_3,Team_4,1-10,ENSG00000000419,DPM1,,Prediction 4,Support 4,syn56789,Study_3,"Protein, Clinical",Validation 4,2018.0
+Source_4,Team_5,,ENSG00000000457,SCYL3,Justification 5,Prediction 5,,,Study_3,RNA,Validation 5,

tests/test_assets/gene_info/input/tep_adi_info_good_input.csv

@@ -0,0 +1,6 @@
+ensembl_gene_id,hgnc_symbol,is_adi,is_tep
+ENSG00000000005,TNMD,True,True
+ENSG00000000419,DPM1,,False
+ENSG00000001497,LAS1L,True,
+ENSG00000001084,GCLC,,True
+ENSG00000183791,ABCD,,

tests/test_assets/gene_info/input/tep_adi_info_type_error_1.csv

@@ -0,0 +1,3 @@
+ensembl_gene_id,hgnc_symbol,is_adi,is_tep
+ENSG00000000005,,True,True
+ENSG00000000419,DPM1,,False

tests/test_assets/gene_info/input/tep_adi_info_type_error_2.csv

@@ -0,0 +1,3 @@
+ensembl_gene_id,hgnc_symbol,is_adi,is_tep
+ENSG00000000005,TNMD,string_value,True
+ENSG00000000419,DPM1,,False

tests/test_assets/gene_info/input/tep_adi_info_type_error_3.csv

@@ -0,0 +1,3 @@
+ensembl_gene_id,hgnc_symbol,is_adi,is_tep
+ENSG00000000005,TNMD,True,string_value
+ENSG00000000419,DPM1,,False