From 1faed03a734db8fd9d2ddd9410ab328f1494ab2d Mon Sep 17 00:00:00 2001 From: Atharva Rane <41084525+atharvar28@users.noreply.github.com> Date: Fri, 17 May 2024 18:02:04 -0400 Subject: [PATCH 1/9] Testing : new alias creation jenkins_subject_alias_new `jenkins_subject_alias_new` for testing the exploration page with '_subject_id' instead of 'subject_id' --- jenkins-dcp.planx-pla.net/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-dcp.planx-pla.net/manifest.json b/jenkins-dcp.planx-pla.net/manifest.json index 9cccac2f2..7041d8d6f 100644 --- a/jenkins-dcp.planx-pla.net/manifest.json +++ b/jenkins-dcp.planx-pla.net/manifest.json @@ -291,7 +291,7 @@ "guppy": { "indices": [ { - "index": "jenkins_subject_alias", + "index": "jenkins_subject_alias_new", "type": "subject" }, { From 8069290e4a2fb43b1b1ae6f02d25dc9651e7a5c2 Mon Sep 17 00:00:00 2001 From: Atharva Rane Date: Mon, 20 May 2024 10:31:51 -0400 Subject: [PATCH 2/9] updating etlMapping and manifest.json --- jenkins-dcp.planx-pla.net/etlMapping.yaml | 222 +++++++++++----------- jenkins-dcp.planx-pla.net/manifest.json | 2 +- 2 files changed, 114 insertions(+), 110 deletions(-) diff --git a/jenkins-dcp.planx-pla.net/etlMapping.yaml b/jenkins-dcp.planx-pla.net/etlMapping.yaml index a93d19f07..2abad9ea8 100644 --- a/jenkins-dcp.planx-pla.net/etlMapping.yaml +++ b/jenkins-dcp.planx-pla.net/etlMapping.yaml @@ -6,115 +6,74 @@ mappings: props: - name: submitter_id - name: project_id - - name: consent_codes - - name: geographic_site - flatten_props: - - path: demographics - props: - - name: age_at_index - - name: hispanic_subgroup - - name: subcohort - - name: weight_baseline - - name: bmi_baseline - - name: height_baseline - - path: medical_histories - props: - - name: diabetes - - name: hypertension - - name: cerebrovascular_disease - - name: chronic_respiratory_disease - - name: coronary_artery_disease - - name: asthma - - name: cabg_presence - - name: copd - - name: emphysema - - name: heart_failure - - name: myocardial_infarction - - name: stroke - - name: cac_score - - name: cac_volume - - name: carotid_plaque - - name: carotid_stenosis - - name: cimt_1 - - name: cimt_2 - - name: vte_case_status - - name: vte_followup_start_age - - name: vte_prior_history - - path: blood_pressure_test - props: - - name: bp_systolic - - name: bp_diastolic - sorted_by: updated_datetime, desc - - path: medication_records - props: - - name: hypertension_meds - - name: antihypertensive_meds - - name: fasting_lipids - - name: lipid_lowering_medication - - path: lab_results - props: - - name: lymphocyte_ncnc_bld - - name: mch_entmass_rbc - - name: mcv_entvol_rbc - - name: mchc_mcnc_rbc - - name: monocyte_ncnc_bld - - name: neutrophil_ncnc_bld - - name: pmv_entvol_bld - - name: rbc_ncnc_bld - - name: rdw_ratio_rbc - - name: hemoglobin_mcnc_bld - - name: platelet_ncnc_bld - - name: hematocrit_vfr_bld - - name: wbc_ncnc_bld - - name: basophil_ncnc_bld - - name: eosinophil_ncnc_bld - - name: unit_triglycerides - - name: hdl - - name: ldl - - name: triglycerides - - name: total_cholesterol + - name: sex + - name: age_value + - name: ancestry + - name: disease_description + - name: phenotype_present + - name: phenotype_absent + - name: disease_id + - name: solve_state + - name: congenital_status + - name: age_of_onset + - name: phenotype_group + - name: anvil_project_id +# flatten_props: aggregated_props: - - name: annotated_sex - path: demographics - src: annotated_sex + - name: sample_provider + path: samples + src: sample_provider + fn: set + - name: tissue_affected_status + path: samples + src: tissue_affected_status + fn: set + - name: tissue_type + path: samples + src: tissue_type + fn: set + - name: sample_type + path: samples + src: sample_type + fn: set + - name: original_material_type + path: samples + src: original_material_type + fn: set + - name: exome_capture_platform + path: samples.sequencings + src: exome_capture_platform + fn: set + - name: library_prep_kit_method + path: samples.sequencings + src: library_prep_kit_method + fn: set + - name: capture_region_bed_file + path: samples.sequencings + src: capture_region_bed_file fn: set - - name: race - path: demographics - src: race + - name: reference_genome_build + path: samples.sequencings + src: reference_genome_build fn: set - - name: ethnicity - path: demographics - src: ethnicity + - name: sequencing_assay + path: samples.sequencings + src: sequencing_assay + fn: set + - name: alignment_method + path: samples.sequencings + src: alignment_method + fn: set + - name: data_processing_pipeline + path: samples.sequencings + src: data_processing_pipeline fn: set - name: _samples_count path: samples fn: count - - name: _aliquots_count - path: samples.aliquots - fn: count - - name: _read_groups_count - path: samples.aliquots.read_groups - fn: count - - name: _submitted_unaligned_reads_files_count - path: samples.aliquots.read_groups.submitted_unaligned_reads_files - fn: count - - name: _submitted_aligned_reads_files_count - path: samples.aliquots.read_groups.submitted_aligned_reads_files + - name: _sequencings_count + path: samples.sequencings fn: count - - name: _simple_germline_variations_count - path: samples.aliquots.read_groups.simple_germline_variations - fn: count - - name: current_smoker_baseline - path: exposures - src: current_smoker_baseline - fn: set - - name: ever_smoker_baseline - path: exposures - src: ever_smoker_baseline - fn: set - parent_props: - - path: studies[studies_submitter_id:submitter_id].projects[code] - - path: studies[studies_submitter_id:submitter_id].projects[code].programs[programs_name:name] joining_props: - index: file join_on: _subject_id @@ -125,25 +84,37 @@ mappings: - name: data_type src: data_type fn: set + - name: data_category + src: data_category + fn: set - name: file_count src: _file_id fn: count + parent_props: + - path: projects[project_code:code] + - path: projects[project_name:name] + - path: projects[project_dbgap_phs:dbgap_phs] + - path: projects[project_dbgap_consent_text:dbgap_consent_text] + - path: projects[project_short_name:short_name] + - path: projects[project_dbgap_accession_number:dbgap_accession_number] - name: jenkins-dcp.planx-pla.net_file doc_type: file type: collector root: None category: data_file props: + - name: project_id + - name: submitter_id - name: object_id - name: md5sum - name: file_name - name: file_size - - name: data_category - name: data_format - name: data_type - name: state - - name: callset - - name: bucket_path + - name: data_category + - name: analyte_type + - name: sequencing_assay - name: source_node injecting_props: subject: @@ -151,9 +122,42 @@ mappings: - name: _subject_id src: id fn: set - - name: project_id - program: - props: - - name: programs_name - src: name + - name: subject_submitter_id + src: submitter_id + fn: set + - name: anvil_project_id + src: anvil_project_id + fn: set + - name: sex + src: sex + fn: set + - name: age_value + src: age_value + fn: set + - name: ancestry + src: ancestry + fn: set + - name: disease_description + src: disease_description + fn: set + - name: phenotype_present + src: phenotype_present + fn: set + - name: phenotype_absent + src: phenotype_absent + fn: set + - name: disease_id + src: disease_id + fn: set + - name: solve_state + src: solve_state + fn: set + - name: congenital_status + src: congenital_status + fn: set + - name: age_of_onset + src: age_of_onset fn: set + - name: phenotype_group + src: phenotype_group + fn: set \ No newline at end of file diff --git a/jenkins-dcp.planx-pla.net/manifest.json b/jenkins-dcp.planx-pla.net/manifest.json index 7041d8d6f..02d1aa522 100644 --- a/jenkins-dcp.planx-pla.net/manifest.json +++ b/jenkins-dcp.planx-pla.net/manifest.json @@ -270,7 +270,7 @@ "environment": "qaplanetv1", "hostname": "jenkins-dcp.planx-pla.net", "revproxy_arn": "arn:aws:acm:us-east-1:707767160287:certificate/c676c81c-9546-4e9a-9a72-725dd3912bc8", - "dictionary_url": "https://s3.amazonaws.com/dictionary-artifacts/gtexdictionary/master/schema.json", + "dictionary_url": "https://s3.amazonaws.com/dictionary-artifacts/anvil/master/schema.json", "portal_app": "gitops", "kube_bucket": "kube-qaplanetv1-gen3", "logs_bucket": "logs-qaplanetv1-gen3", From ddf2200a0cd9c35f6ef096917fec2115a63ee1cf Mon Sep 17 00:00:00 2001 From: Atharva Rane Date: Tue, 21 May 2024 18:16:18 -0400 Subject: [PATCH 3/9] update gitops.json --- jenkins-dcp.planx-pla.net/portal/gitops.json | 223 +------------------ 1 file changed, 4 insertions(+), 219 deletions(-) diff --git a/jenkins-dcp.planx-pla.net/portal/gitops.json b/jenkins-dcp.planx-pla.net/portal/gitops.json index 6f585b54c..d1d7f7f38 100644 --- a/jenkins-dcp.planx-pla.net/portal/gitops.json +++ b/jenkins-dcp.planx-pla.net/portal/gitops.json @@ -608,14 +608,7 @@ "_subject_id": { "chartType": "count", "title": "Subjects" - }, - "annotated_sex": { - "chartType": "pie", - "title": "Annotated Sex" - }, - "race": { - "chartType": "bar", - "title": "Race" + } }, "fileCounts": false }, @@ -626,68 +619,9 @@ "fields": [ "programs_name", "project_id", - "consent_codes", - "studies_submitter_id", - "geographic_site", "data_type", "data_format" ] - }, - { - "title": "Subject", - "searchFields": [ - "submitter_id" - ], - "fields": [ - "annotated_sex", - "race", - "ethnicity", - "hispanic_subgroup", - "subcohort", - "weight_baseline", - "bmi_baseline", - "height_baseline", - "current_smoker_baseline", - "ever_smoker_baseline" - ] - }, - { - "title": "Harmonized Variables", - "fields": [ - "total_cholesterol", - "hdl", - "ldl", - "triglycerides", - "fasting_lipids", - "lipid_lowering_medication", - "cac_score", - "cac_volume", - "carotid_plaque", - "carotid_stenosis", - "cimt_1", - "cimt_2", - "bp_diastolic", - "bp_systolic", - "antihypertensive_meds", - "vte_case_status", - "vte_followup_start_age", - "vte_prior_history", - "basophil_ncnc_bld", - "eosinophil_ncnc_bld", - "hematocrit_vfr_bld", - "hemoglobin_mcnc_bld", - "lymphocyte_ncnc_bld", - "mch_entmass_rbc", - "mchc_mcnc_rbc", - "mcv_entvol_rbc", - "monocyte_ncnc_bld", - "neutrophil_ncnc_bld", - "platelet_ncnc_bld", - "pmv_entvol_bld", - "rbc_ncnc_bld", - "rdw_ratio_rbc", - "wbc_ncnc_bld" - ] } ] }, @@ -695,13 +629,9 @@ "enabled": true, "fields": [ "project_id", + "file_id", "data_format", - "race", - "annotated_sex", - "ethnicity", - "bp_diastolic", - "hdl", - "ldl" + "breed" ] }, "dropdowns": { @@ -740,156 +670,11 @@ "guppyConfig": { "dataType": "subject", "nodeCountTitle": "Subjects", - "fileCountField": "file_count", "fieldMapping": [ { "field": "_subject_id", "name": "Subject ID" - }, - { - "field": "consent_codes", - "name": "data use restriction" - }, - { - "field": "cac_score", - "name": "CAC Score" - }, - { - "field": "cac_volume", - "name": "CAC Volume" - }, - { - "field": "cimt_1", - "name": "CIMT 1" - }, - { - "field": "cimt_2", - "name": "CIMT 2" - }, - { - "field": "vte_case_status", - "name": "VTE Case Status" - }, - { - "field": "vte_followup_start_age", - "name": "VTE Followup Start Age" - }, - { - "field": "vte_prior_history", - "name": "VTE Prior History" - }, - { - "field": "bp_diastolic", - "name": "BP Diastolic" - }, - { - "field": "bp_systolic", - "name": "BP Systolic" - }, - { - "field": "basophil_ncnc_bld", - "name": "Basophils Number Concentration Bld" - }, - { - "field": "eosinophil_ncnc_bld", - "name": "Eosinophils Number Concentration Bld" - }, - { - "field": "hdl", - "name": "HDL" - }, - { - "field": "hematocrit_vfr_bld", - "name": "Hematocrit Fraction of Volume Bld" - }, - { - "field": "hemoglobin_mcnc_bld", - "name": "Hemoglobin Mass Concentration Bld" - }, - { - "field": "ldl", - "name": "LDL" - }, - { - "field": "lymphocyte_ncnc_bld", - "name": "Lymphocytes Number Concentration Bld" - }, - { - "field": "mch_entmass_rbc", - "name": "MCH Average Mass Rbc" - }, - { - "field": "mchc_mcnc_rbc", - "name": "MCHC Mass Concentration Rbc" - }, - { - "field": "mcv_entvol_rbc", - "name": "MCV Average Volume Rbc" - }, - { - "field": "monocyte_ncnc_bld", - "name": "Monocytes Number Concentration Bld" - }, - { - "field": "neutrophil_ncnc_bld", - "name": "Neutrophils Number Concentration Bld" - }, - { - "field": "platelet_ncnc_bld", - "name": "Platelets Number Concentration Bld" - }, - { - "field": "pmv_entvol_bld", - "name": "PMV Mean Volume Bld" - }, - { - "field": "rbc_ncnc_bld", - "name": "Red Blood Cells Number Concentration Bld" - }, - { - "field": "rdw_ratio_rbc", - "name": "Red Cell Distribution Width Ratio Rbc" - }, - { - "field": "wbc_ncnc_bld", - "name": "White Blood Cells Number Concentration Bld" - }, - { - "field": "bmi_baseline", - "name": "BMI Baseline" - }, - { - "field": "current_smoker_baseline", - "name": "Current Smoker Baseline" - }, - { - "field": "ever_smoker_baseline", - "name": "Ever Smoker Baseline" - }, - { - "field": "geographic_site", - "name": "Geographic Site" - }, - { - "field": "height_baseline", - "name": "Height Baseline" - }, - { - "field": "total_cholesterol", - "name": "Total Cholesterol" - }, - { - "field": "triglycerides", - "name": "Triglycerides" - }, - { - "field": "studies_submitter_id", - "name": "Study" - }, - { - "field": "programs_name", - "name": "Program" - } + } ], "manifestMapping": { "resourceIndexType": "file", From 62fbaf15a8936efbf1522f88c5004ffb348f9938 Mon Sep 17 00:00:00 2001 From: Atharva Rane Date: Tue, 21 May 2024 18:25:47 -0400 Subject: [PATCH 4/9] fix gitop.json --- jenkins-dcp.planx-pla.net/portal/gitops.json | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins-dcp.planx-pla.net/portal/gitops.json b/jenkins-dcp.planx-pla.net/portal/gitops.json index d1d7f7f38..0f160ea53 100644 --- a/jenkins-dcp.planx-pla.net/portal/gitops.json +++ b/jenkins-dcp.planx-pla.net/portal/gitops.json @@ -608,7 +608,6 @@ "_subject_id": { "chartType": "count", "title": "Subjects" - } }, "fileCounts": false }, From c6e87aedeacafd6f0c58a5800dc2502315cbe516 Mon Sep 17 00:00:00 2001 From: Atharva Rane Date: Tue, 28 May 2024 10:52:40 -0400 Subject: [PATCH 5/9] fix --- jenkins-dcp.planx-pla.net/portal/gitops.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jenkins-dcp.planx-pla.net/portal/gitops.json b/jenkins-dcp.planx-pla.net/portal/gitops.json index 0f160ea53..113dd5738 100644 --- a/jenkins-dcp.planx-pla.net/portal/gitops.json +++ b/jenkins-dcp.planx-pla.net/portal/gitops.json @@ -608,15 +608,13 @@ "_subject_id": { "chartType": "count", "title": "Subjects" - }, - "fileCounts": false + } }, "filters": { "tabs": [ { "title": "Project", "fields": [ - "programs_name", "project_id", "data_type", "data_format" From d43e055b27d9f2c8fba07bd4b014d6f9aa191899 Mon Sep 17 00:00:00 2001 From: Atharva Rane <41084525+atharvar28@users.noreply.github.com> Date: Tue, 28 May 2024 15:21:26 -0400 Subject: [PATCH 6/9] Update manifest.json --- jenkins-dcp.planx-pla.net/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-dcp.planx-pla.net/manifest.json b/jenkins-dcp.planx-pla.net/manifest.json index 02d1aa522..838b40828 100644 --- a/jenkins-dcp.planx-pla.net/manifest.json +++ b/jenkins-dcp.planx-pla.net/manifest.json @@ -291,7 +291,7 @@ "guppy": { "indices": [ { - "index": "jenkins_subject_alias_new", + "index": "jenkins_subject_new_alias", "type": "subject" }, { From 9bc4296a118afa174bf1f452cba24661b9139fc4 Mon Sep 17 00:00:00 2001 From: Atharva Rane Date: Mon, 3 Jun 2024 14:50:05 -0400 Subject: [PATCH 7/9] update gitops.json --- jenkins-dcp.planx-pla.net/portal/gitops.json | 24 +++++++------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/jenkins-dcp.planx-pla.net/portal/gitops.json b/jenkins-dcp.planx-pla.net/portal/gitops.json index 113dd5738..aab45d5dd 100644 --- a/jenkins-dcp.planx-pla.net/portal/gitops.json +++ b/jenkins-dcp.planx-pla.net/portal/gitops.json @@ -575,16 +575,14 @@ "titleField": "project_id", "rowAccessor": "submitter_id", "listItemConfig": { - "blockFields": ["breed"], - "tableFields": ["data_type", "auth_resource_path", "disease_type"] + "blockFields": ["project_id"], + "tableFields": ["data_type"] }, "fieldMapping": [ - { "field": "breed", "name": "Breed" }, + { "field": "category", "name": "Category" }, { "field": "project_id", "name": "Project_ID"}, - { "field": "submitter_id", "name": "Submitter_ID"}, { "field": "data_type", "name": "Data_Type"}, - { "field": "auth_resource_path", "name": "Resource Path"}, - { "field": "disease_type", "name": "Disease"} + { "field": "condition", "name": "Condition"} ], "openMode": "close-all", "buttons": [ @@ -626,9 +624,7 @@ "enabled": true, "fields": [ "project_id", - "file_id", - "data_format", - "breed" + "data_format" ] }, "dropdowns": { @@ -708,13 +704,11 @@ { "title": "File", "fields": [ - "programs_name", "project_id", "data_category", "data_type", - "data_format", - "callset", - "bucket_path" + "file_id", + "data_format" ], "searchFields": [ "file_name" @@ -728,9 +722,7 @@ "project_id", "file_name", "file_size", - "object_id", - "callset" - ] + "object_id" ] }, "guppyConfig": { "dataType": "file", From 5e371e77027745a418584ba165516389e95355cc Mon Sep 17 00:00:00 2001 From: Atharva Rane Date: Mon, 3 Jun 2024 14:55:28 -0400 Subject: [PATCH 8/9] fix again --- jenkins-dcp.planx-pla.net/portal/gitops.json | 9 --------- 1 file changed, 9 deletions(-) diff --git a/jenkins-dcp.planx-pla.net/portal/gitops.json b/jenkins-dcp.planx-pla.net/portal/gitops.json index aab45d5dd..04bd4bcb6 100644 --- a/jenkins-dcp.planx-pla.net/portal/gitops.json +++ b/jenkins-dcp.planx-pla.net/portal/gitops.json @@ -55,10 +55,6 @@ { "node": "subject", "name": "Subjects" - }, - { - "node": "study", - "name": "Studies" } ] }, @@ -602,10 +598,6 @@ "project_id": { "chartType": "count", "title": "Projects" - }, - "_subject_id": { - "chartType": "count", - "title": "Subjects" } }, "filters": { @@ -707,7 +699,6 @@ "project_id", "data_category", "data_type", - "file_id", "data_format" ], "searchFields": [ From a523b320e8c3b582b20fdb300bc9743adfeb5459 Mon Sep 17 00:00:00 2001 From: Atharva Rane Date: Wed, 12 Jun 2024 10:21:24 -0400 Subject: [PATCH 9/9] update the gitops.json --- jenkins-dcp.planx-pla.net/portal/gitops.json | 63 ++++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/jenkins-dcp.planx-pla.net/portal/gitops.json b/jenkins-dcp.planx-pla.net/portal/gitops.json index 04bd4bcb6..6c2a745db 100644 --- a/jenkins-dcp.planx-pla.net/portal/gitops.json +++ b/jenkins-dcp.planx-pla.net/portal/gitops.json @@ -598,6 +598,14 @@ "project_id": { "chartType": "count", "title": "Projects" + }, + "sex": { + "chartType": "pie", + "title": "Sex" + }, + "ancestry": { + "chartType": "bar", + "title": "Ancestry" } }, "filters": { @@ -606,9 +614,46 @@ "title": "Project", "fields": [ "project_id", + "anvil_project_id", "data_type", "data_format" ] + }, + { + "title": "Subject", + "fields":[ + "sex", + "ancestry", + "age_value", + "disease_description", + "phenotype_present", + "phenotype_absent", + "disease_id", + "solve_state", + "congenital_status", + "age_of_onset", + "phenotype_group" + ] + }, { + "title": "Sample", + "fields": [ + "tissue_type", + "sample_type", + "original_material_type", + "sample_provider", + "tissue_affected_status" + ] + }, { + "title": "Sequencing", + "fields": [ + "sequencing_assay", + "reference_genome_build", + "library_prep_kit_method", + "exome_capture_platform", + "capture_region_bed_file", + "alignment_method", + "data_processing_pipeline" + ] } ] }, @@ -616,7 +661,13 @@ "enabled": true, "fields": [ "project_id", - "data_format" + "anvil_project_id", + "ancestry", + "sex", + "age_value", + "phenotype_group", + "_samples_count", + "_sequencings_count" ] }, "dropdowns": { @@ -655,11 +706,13 @@ "guppyConfig": { "dataType": "subject", "nodeCountTitle": "Subjects", + "fileCountField": "file_count", "fieldMapping": [ - { - "field": "_subject_id", - "name": "Subject ID" - } + { "field": "disease_id", "name": "Disease ID" }, + { "field": "age_of_onset", "name": "Age of Onset" }, + { "field": "project_dbgap_accession_number", "name": "Project dbGaP Accession Number" }, + { "field": "project_dbgap_consent_text", "name":"Project dbGaP Consent Text"}, + { "field": "project_dbgap_phs", "name":"Project dbGaP Phs"} ], "manifestMapping": { "resourceIndexType": "file",