[IBCDPE-796] Adds rnaseq_differential_expression expectation suite (#…

…136)

* adds rnaseq_differential_expression expectation suite

* clean up jupyter notebook

* pre-commit

* updates dev_config gitignore

config.yaml

@@ -254,6 +254,7 @@ datasets:
       custom_transformations: 1
       provenance: *rna_diff_expr_data_provenance
       destination: *dest
+      gx_enabled: true
 
   - distribution_data:
       files: *overall_scores_files

gx_suite_definitions/rnaseq_differential_expression.ipynb

@@ -0,0 +1,282 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import synapseclient\n",
+    "\n",
+    "import great_expectations as gx\n",
+    "\n",
+    "context = gx.get_context(project_root_dir='../src/agoradatatools/great_expectations')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Expectation Suite for RNASEQ Differential Expression Dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Get Example Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "syn = synapseclient.Synapse()\n",
+    "syn.login()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rnaseq_de_data_file = syn.get(\"syn17015360\").path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Validator Object on Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "validator = context.sources.pandas_default.read_json(\n",
+    "    rnaseq_de_data_file\n",
+    ")\n",
+    "validator.expectation_suite_name = \"rnaseq_differential_expression\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Add Expectations to Validator Object For Each Column"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ensembl_gene_id\n",
+    "validator.expect_column_values_to_be_of_type(\"ensembl_gene_id\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"ensembl_gene_id\")\n",
+    "validator.expect_column_value_lengths_to_equal(\"ensembl_gene_id\", 15)\n",
+    "validator.expect_column_values_to_match_regex(\"ensembl_gene_id\", \"^ENSG\\d{11}$\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# hgnc_symbol\n",
+    "validator.expect_column_values_to_be_of_type(\"hgnc_symbol\", \"str\")\n",
+    "validator.expect_column_value_lengths_to_be_between(\"hgnc_symbol\", 1, 25)\n",
+    "validator.expect_column_values_to_match_regex(\"hgnc_symbol\", \"^[a-zA-Z0-9_.-]*$\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# logfc\n",
+    "validator.expect_column_values_to_be_of_type(\"logfc\", \"float\")\n",
+    "validator.expect_column_values_to_not_be_null(\"logfc\")\n",
+    "validator.expect_column_values_to_be_between(\"logfc\", -5, 5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# fc\n",
+    "validator.expect_column_values_to_be_of_type(\"fc\", \"float\")\n",
+    "validator.expect_column_values_to_not_be_null(\"fc\")\n",
+    "validator.expect_column_values_to_be_between(\"fc\", -1, 7)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ci_l\n",
+    "validator.expect_column_values_to_be_of_type(\"ci_l\", \"float\")\n",
+    "validator.expect_column_values_to_not_be_null(\"ci_l\")\n",
+    "validator.expect_column_values_to_be_between(\"ci_l\", -5, 5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# ci_r\n",
+    "validator.expect_column_values_to_be_of_type(\"ci_r\", \"float\")\n",
+    "validator.expect_column_values_to_not_be_null(\"ci_r\")\n",
+    "validator.expect_column_values_to_be_between(\"ci_r\", -5, 5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# adj_p_val\n",
+    "validator.expect_column_values_to_be_of_type(\"adj_p_val\", \"float\")\n",
+    "validator.expect_column_values_to_not_be_null(\"adj_p_val\")\n",
+    "validator.expect_column_values_to_be_between(\"adj_p_val\", 0, 1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# tissue\n",
+    "validator.expect_column_values_to_be_of_type(\"tissue\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"tissue\")\n",
+    "validator.expect_column_values_to_be_in_set(\"tissue\", [\"CBE\", \"TCX\", \"FP\", \"IFG\", \"PHG\", \"STG\", \"DLPFC\", \"ACC\", \"PCC\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# study\n",
+    "validator.expect_column_values_to_be_of_type(\"study\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"study\")\n",
+    "validator.expect_column_values_to_be_in_set(\"study\", [\"MayoRNAseq\", \"MSBB\", \"ROSMAP\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# model\n",
+    "validator.expect_column_values_to_be_of_type(\"model\", \"str\")\n",
+    "validator.expect_column_values_to_be_of_type(\"model\", \"str\")\n",
+    "validator.expect_column_values_to_be_in_set(\"model\", [\"AD Diagnosis (males and females)\", \"AD Diagnosis x AOD (males and females)\", \"AD Diagnosis x Sex (females only)\", \"AD Diagnosis x Sex (males only)\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# multi-field\n",
+    "validator.expect_compound_columns_to_be_unique([\"ensembl_gene_id\", \"tissue\", \"model\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Save Expectation Suite"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "validator.save_expectation_suite(discard_failed_expectations=False)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Checkpoint and View Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "checkpoint = context.add_or_update_checkpoint(\n",
+    "    name=\"agora-test-checkpoint\",\n",
+    "    validator=validator,\n",
+    ")\n",
+    "checkpoint_result = checkpoint.run()\n",
+    "context.view_validation_result(checkpoint_result)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build Data Docs - Click on Expectation Suite to View All Expectations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "context.build_data_docs()\n",
+    "context.open_data_docs()\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}