[IBCDPE-793] Implement Great Expectations for the proteomics_tmt Dataset

config.yaml

@@ -100,6 +100,7 @@ datasets:
         genename: hgnc_symbol
         ensg: ensembl_gene_id
       destination: *dest
+      gx_folder: syn53469660
 
   - proteomics_srm:
       files: *agora_proteomics_srm_files

gx_suite_definitions/proteomics_tmt.ipynb

@@ -0,0 +1,293 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import synapseclient\n",
+    "\n",
+    "import pandas as pd\n",
+    "import great_expectations as gx\n",
+    "\n",
+    "from agoradatatools.gx import GreatExpectationsRunner\n",
+    "\n",
+    "context = gx.get_context(project_root_dir='../src/agoradatatools/great_expectations')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Expectation Suite for Proteomics TMT Data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Get Example Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "syn = synapseclient.Synapse()\n",
+    "syn.login()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "proteomics_tmt_file = syn.get(\"syn32210527\").path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Validator Object on Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_json(proteomics_tmt_file)\n",
+    "nested_columns = []\n",
+    "df = GreatExpectationsRunner.convert_nested_columns_to_json(df, nested_columns)\n",
+    "validator = context.sources.pandas_default.read_dataframe(df)\n",
+    "validator.expectation_suite_name = \"proteomics_tmt\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Add Expectations to Validator Object For Each Column"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# uniqid\n",
+    "validator.expect_column_values_to_be_of_type(\"uniqid\", \"str\")\n",
+    "validator.expect_column_value_lengths_to_be_between(\"uniqid\", 1, 25)\n",
+    "validator.expect_column_values_to_match_regex(\"uniqid\", \"^[a-zA-Z0-9.]+?|[a-zA-Z0-9-]+$\")\n",
+    "validator.expect_column_values_to_be_unique(\"uniqid\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hgnc_symbol\n",
+    "validator.expect_column_values_to_be_of_type(\"hgnc_symbol\", \"str\")\n",
+    "validator.expect_column_value_lengths_to_be_between(\"hgnc_symbol\", 1, 15)\n",
+    "validator.expect_column_values_to_match_regex(\"hgnc_symbol\", \"^[a-zA-Z0-9.-]*$\")\n",
+    "validator.expect_column_values_to_not_be_null(\"hgnc_symbol\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# uniprotid\n",
+    "validator.expect_column_values_to_be_of_type(\"uniprotid\", \"str\")\n",
+    "validator.expect_column_value_lengths_to_be_between(\"uniprotid\", 1, 15)\n",
+    "validator.expect_column_values_to_match_regex(\"uniprotid\", \"^[a-zA-Z0-9.-]*$\")\n",
+    "validator.expect_column_values_to_be_unique(\"uniprotid\")\n",
+    "validator.expect_column_values_to_not_be_null(\"uniprotid\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ensembl_gene_id\n",
+    "validator.expect_column_values_to_be_of_type(\"ensembl_gene_id\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"ensembl_gene_id\")\n",
+    "validator.expect_column_value_lengths_to_equal(\"ensembl_gene_id\", 15)\n",
+    "# checks format and allowed chatacters\n",
+    "validator.expect_column_values_to_match_regex(\"ensembl_gene_id\", \"^ENSG\\d{11}$\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# tissue\n",
+    "validator.expect_column_values_to_be_of_type(\"tissue\", \"str\")\n",
+    "validator.expect_column_value_lengths_to_be_between(\"tissue\", 1, 15)\n",
+    "validator.expect_column_values_to_be_in_set(\"tissue\", [\"AntPFC\", \"DLPFC\", \"MFG\", \"TCX\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# log2_fc\n",
+    "validator.expect_column_values_to_be_of_type(\"log2_fc\", \"float\")\n",
+    "validator.expect_column_values_to_be_between(\"log2_fc\", -0.5, 1.5)\n",
+    "validator.expect_column_values_to_not_be_null(\"log2_fc\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ci_upr\n",
+    "validator.expect_column_values_to_be_of_type(\"ci_upr\", \"float\")\n",
+    "validator.expect_column_values_to_be_between(\"ci_upr\", -0.5, 2)\n",
+    "validator.expect_column_values_to_not_be_null(\"ci_upr\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ci_lwr\n",
+    "validator.expect_column_values_to_be_of_type(\"ci_lwr\", \"float\")\n",
+    "validator.expect_column_values_to_be_between(\"ci_lwr\", -1, 1.5)\n",
+    "validator.expect_column_values_to_not_be_null(\"ci_lwr\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# pval\n",
+    "validator.expect_column_values_to_be_of_type(\"pval\", \"float\")\n",
+    "validator.expect_column_values_to_be_between(\"pval\", 0, 1)\n",
+    "validator.expect_column_values_to_not_be_null(\"pval\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# cor_pval\n",
+    "validator.expect_column_values_to_be_of_type(\"cor_pval\", \"float\")\n",
+    "validator.expect_column_values_to_be_between(\"cor_pval\", 0, 1)\n",
+    "validator.expect_column_values_to_not_be_null(\"cor_pval\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# multi-field\n",
+    "validator.expect_column_pair_values_a_to_be_greater_than_b(\"ci_upr\", \"ci_lwr\")\n",
+    "validator.expect_compound_columns_to_be_unique([\"uniqid\", \"tissue\"])\n",
+    "validator.expect_compound_columns_to_be_unique([\"hgnc_symbol\", \"uniprotid\", \"tissue\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Save Expectation Suite"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "validator.save_expectation_suite(discard_failed_expectations=False)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Checkpoint and View Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "checkpoint = context.add_or_update_checkpoint(\n",
+    "    name=\"agora-test-checkpoint\",\n",
+    "    validator=validator,\n",
+    ")\n",
+    "checkpoint_result = checkpoint.run()\n",
+    "context.view_validation_result(checkpoint_result)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build Data Docs - Click on Expectation Suite to View All Expectations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "context.build_data_docs()\n",
+    "context.open_data_docs()\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "agora-data-tools-CK0oUlHB",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}