[IBCDPE-766] Adds distribution_data GX suite (#140)

* adds distribution data gx suite

* updates config files

config.yaml

@@ -275,6 +275,11 @@ datasets:
         geneticsscore: genetics_score
         omicsscore: multi_omics_score
       destination: *dest
+      gx_enabled: true
+      gx_nested_columns:
+        - target_risk_score
+        - genetics_score
+        - multi_omics_score
 
   - rna_distribution_data:
       files: *rna_diff_expr_data_files

gx_suite_definitions/distribution_data.ipynb

@@ -0,0 +1,201 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import synapseclient\n",
+    "import json\n",
+    "\n",
+    "import pandas as pd\n",
+    "import great_expectations as gx\n",
+    "\n",
+    "from agoradatatools.gx import GreatExpectationsRunner\n",
+    "\n",
+    "context = gx.get_context(project_root_dir='../src/agoradatatools/great_expectations')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Expectation Suite for Distribution Data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Get Example Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "syn = synapseclient.Synapse()\n",
+    "syn.login()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "distribution_data_file = syn.get(\"syn27572407\").path\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Validator Object on Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_json(distribution_data_file)\n",
+    "nested_columns = ['target_risk_score', 'genetics_score', 'multi_omics_score']\n",
+    "df = GreatExpectationsRunner.convert_nested_columns_to_json(df, nested_columns)\n",
+    "validator = context.sources.pandas_default.read_dataframe(df)\n",
+    "validator.expectation_suite_name = \"distribution_data\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Add Expectations to Validator Object For Each Column"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# target_risk_score\n",
+    "validator.expect_column_values_to_be_of_type(\"target_risk_score\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"target_risk_score\")\n",
+    "#get JSON schema\n",
+    "with open(\"../src/agoradatatools/great_expectations/gx/json_schemas/distribution_data/target_risk_score.json\", \"r\") as file:\n",
+    "    target_risk_score_schema = json.load(file)\n",
+    "validator.expect_column_values_to_match_json_schema(\"target_risk_score\", json_schema=target_risk_score_schema)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# genetics_score\n",
+    "validator.expect_column_values_to_be_of_type(\"genetics_score\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"genetics_score\")\n",
+    "#get JSON schema\n",
+    "with open(\"../src/agoradatatools/great_expectations/gx/json_schemas/distribution_data/genetics_score.json\", \"r\") as file:\n",
+    "    genetics_score_schema = json.load(file)\n",
+    "validator.expect_column_values_to_match_json_schema(\"genetics_score\", json_schema=genetics_score_schema)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# multi_omics_score\n",
+    "validator.expect_column_values_to_be_of_type(\"multi_omics_score\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"multi_omics_score\")\n",
+    "#get JSON schema\n",
+    "with open(\"../src/agoradatatools/great_expectations/gx/json_schemas/distribution_data/multi_omics_score.json\", \"r\") as file:\n",
+    "    multi_omics_score_schema = json.load(file)\n",
+    "validator.expect_column_values_to_match_json_schema(\"multi_omics_score\", json_schema=multi_omics_score_schema)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Save Expectation Suite"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "validator.save_expectation_suite(discard_failed_expectations=False)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Checkpoint and View Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "checkpoint = context.add_or_update_checkpoint(\n",
+    "    name=\"agora-test-checkpoint\",\n",
+    "    validator=validator,\n",
+    ")\n",
+    "checkpoint_result = checkpoint.run()\n",
+    "context.view_validation_result(checkpoint_result)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build Data Docs - Click on Expectation Suite to View All Expectations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "context.build_data_docs()\n",
+    "context.open_data_docs()\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}