2.3.1_make_table_for_r.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "acf2b930",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import scipy.stats\n",
    "from collections import Counter\n",
    "GEST = 'out_data/cor_full_GEST_DIABETES1.txt'\n",
    "HYPT = 'out_data/cor_full_I9_HYPTENSPREG1.txt'\n",
    "PRET = 'out_data/cor_full_O15_PRETERM1.txt'\n",
    "FINN_UK = 'data/finn_uk_codes.csv'\n",
    "PRIOR_LIST = ['2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bc3b79d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>full_name</th>\n",
       "      <th>priority</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>E4_THYROID___C73</th>\n",
       "      <td>FG: Disorders of the thyroid gland &amp; \\nUK: Mal...</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C3_THYROID_GLAND___C73</th>\n",
       "      <td>FG: Malignant neoplasm of thyroid gland &amp; \\nUK...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C3_THYROID_GLAND_EXALLC___C73</th>\n",
       "      <td>FG: Malignant neoplasm of thyroid gland (all c...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_THYTOXNOD___C73</th>\n",
       "      <td>FG: Thyrotoxicosis with toxic single thyroid n...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_RHEUVALV___I35</th>\n",
       "      <td>FG: Rheumatic valve diseases &amp; \\nUK: Nonrheuma...</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>L12_LUPUS___M32</th>\n",
       "      <td>FG: Lupus erythematosus &amp; \\nUK: Systemic lupus...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M13_SLE___M32</th>\n",
       "      <td>FG: Systemic lupus erythematosus &amp; \\nUK: Syste...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SLE_NOS___M32</th>\n",
       "      <td>FG: Systemic lupus erythematosus, unspecified ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SLE_OTH___M32</th>\n",
       "      <td>FG: Other forms of systemic lupus erythematosu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DRUGADVERS_SYSTEMIC_LUPUS_ERYTHEMAT___M32</th>\n",
       "      <td>FG: Drug-induced systemic lupus erythematosus ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1671 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                   full_name  \\\n",
       "E4_THYROID___C73                           FG: Disorders of the thyroid gland & \\nUK: Mal...   \n",
       "C3_THYROID_GLAND___C73                     FG: Malignant neoplasm of thyroid gland & \\nUK...   \n",
       "C3_THYROID_GLAND_EXALLC___C73              FG: Malignant neoplasm of thyroid gland (all c...   \n",
       "E4_THYTOXNOD___C73                         FG: Thyrotoxicosis with toxic single thyroid n...   \n",
       "I9_RHEUVALV___I35                          FG: Rheumatic valve diseases & \\nUK: Nonrheuma...   \n",
       "...                                                                                      ...   \n",
       "L12_LUPUS___M32                            FG: Lupus erythematosus & \\nUK: Systemic lupus...   \n",
       "M13_SLE___M32                              FG: Systemic lupus erythematosus & \\nUK: Syste...   \n",
       "SLE_NOS___M32                              FG: Systemic lupus erythematosus, unspecified ...   \n",
       "SLE_OTH___M32                              FG: Other forms of systemic lupus erythematosu...   \n",
       "DRUGADVERS_SYSTEMIC_LUPUS_ERYTHEMAT___M32  FG: Drug-induced systemic lupus erythematosus ...   \n",
       "\n",
       "                                          priority  \n",
       "E4_THYROID___C73                                 *  \n",
       "C3_THYROID_GLAND___C73                           2  \n",
       "C3_THYROID_GLAND_EXALLC___C73                    1  \n",
       "E4_THYTOXNOD___C73                               0  \n",
       "I9_RHEUVALV___I35                                *  \n",
       "...                                            ...  \n",
       "L12_LUPUS___M32                                  2  \n",
       "M13_SLE___M32                                    2  \n",
       "SLE_NOS___M32                                    2  \n",
       "SLE_OTH___M32                                    1  \n",
       "DRUGADVERS_SYSTEMIC_LUPUS_ERYTHEMAT___M32        0  \n",
       "\n",
       "[1671 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fu = pd.read_csv(FINN_UK)\n",
    "fu = fu\n",
    "fu.index = fu['0']+ '___' +fu['UK_code']\n",
    "fu['full_name'] = 'FG: ' + fu['finn_pheno'] + ' & \\nUK: ' + fu['UK_pheno']\n",
    "fu = fu.loc[:,['full_name', 'priority']]\n",
    "fu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dfe6e521",
   "metadata": {},
   "outputs": [],
   "source": [
    "ph_map = {'I9_HYPTENSPREG':'HP', 'GEST_DIABETES':'GD', 'O15_PRETERM':'PTB'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dd6fecef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def return_df(FILE):\n",
    "    data = ''\n",
    "    with open(FILE, 'r') as in_f:\n",
    "        data = in_f.read().replace('\\n--', '').split('\\n')\n",
    "    keys = list(map(lambda x: x.split('/')[0].replace('analysis_n_', ''), data[::2]))\n",
    "    values = map(lambda x: x.split(' '), data[1::2])\n",
    "    traits = map(lambda x: x.split('/')[-1].replace('1.cors', ''), data[::2])\n",
    "    data_dict = {k: v for k, v in zip(keys, values)}\n",
    "    for k, v in zip(keys, traits):\n",
    "        try:\n",
    "            data_dict[k].append(v)\n",
    "        except KeyError:\n",
    "            print(k)\n",
    "    return pd.DataFrame(data_dict, index=['Cor_All', 'cor', 'cor_std', 'trait']).T\n",
    "\n",
    "\n",
    "def count_sign(data):\n",
    "    data['grep_stat'] = data.cor.astype(float)/data.cor_std.astype(float)\n",
    "    data['pval'] = scipy.stats.norm.sf(abs(data['grep_stat']))*2\n",
    "    data['p2'] = data.index\n",
    "    PVAL_CUTOFF = 0.05/data.shape[0]\n",
    "    data['significant'] = ''\n",
    "    data.loc[data.pval<PVAL_CUTOFF, ['significant']] = '*'\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6fee19c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# g, h = list(gest_data.index), list(hypt_data.index)\n",
    "# len(g), len(h), len(set(g)), len(set(h)), len(set(g) & set(h))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e8122e45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cor_All</th>\n",
       "      <th>cor</th>\n",
       "      <th>cor_std</th>\n",
       "      <th>trait</th>\n",
       "      <th>grep_stat</th>\n",
       "      <th>pval</th>\n",
       "      <th>p2</th>\n",
       "      <th>significant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AB1_BACT_INTEST_OTH___A04</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.434338</td>\n",
       "      <td>0.447854</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>0.969821</td>\n",
       "      <td>0.332136</td>\n",
       "      <td>AB1_BACT_INTEST_OTH___A04</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_CANDIDIASIS___B37</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>-0.330457</td>\n",
       "      <td>0.529697</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>-0.623860</td>\n",
       "      <td>0.532719</td>\n",
       "      <td>AB1_CANDIDIASIS___B37</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_GASTROENTERITIS_NOS___A09</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>1.082440</td>\n",
       "      <td>0.836931</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>1.293344</td>\n",
       "      <td>0.195892</td>\n",
       "      <td>AB1_GASTROENTERITIS_NOS___A09</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_OTHER_SEPSIS___A41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.306278</td>\n",
       "      <td>0.729840</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>0.419651</td>\n",
       "      <td>0.674741</td>\n",
       "      <td>AB1_OTHER_SEPSIS___A41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_SEPSIS___A41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.227972</td>\n",
       "      <td>1.581424</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>0.144156</td>\n",
       "      <td>0.885377</td>\n",
       "      <td>AB1_SEPSIS___A41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.245154</td>\n",
       "      <td>0.342948</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>0.714843</td>\n",
       "      <td>0.474706</td>\n",
       "      <td>Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_REPOR_DIAGNOSIS___Z01</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.788314</td>\n",
       "      <td>0.305386</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>2.581369</td>\n",
       "      <td>0.009841</td>\n",
       "      <td>Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PROCED_NOT_CARRIED___Z53</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.663892</td>\n",
       "      <td>0.433588</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>1.531159</td>\n",
       "      <td>0.125730</td>\n",
       "      <td>Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_P...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.411307</td>\n",
       "      <td>0.389700</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>1.055445</td>\n",
       "      <td>0.291222</td>\n",
       "      <td>Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_SPECIAL_SCREEN_EXAM_INFECTIOUS_PARASITIC_DISEA___Z11</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>-0.072549</td>\n",
       "      <td>1.087701</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>-0.066699</td>\n",
       "      <td>0.946821</td>\n",
       "      <td>Z21_SPECIAL_SCREEN_EXAM_INFECTIOUS_PARASITIC_D...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>441 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Cor_All        cor  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           Cor_All   0.434338   \n",
       "AB1_CANDIDIASIS___B37                               Cor_All  -0.330457   \n",
       "AB1_GASTROENTERITIS_NOS___A09                       Cor_All   1.082440   \n",
       "AB1_OTHER_SEPSIS___A41                              Cor_All   0.306278   \n",
       "AB1_SEPSIS___A41                                    Cor_All   0.227972   \n",
       "...                                                     ...        ...   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                Cor_All   0.245154   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  Cor_All   0.788314   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  Cor_All   0.663892   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41       Cor_All   0.411307   \n",
       "Z21_SPECIAL_SCREEN_EXAM_INFECTIOUS_PARASITIC_DI...  Cor_All  -0.072549   \n",
       "\n",
       "                                                     cor_std          trait  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           0.447854  GEST_DIABETES   \n",
       "AB1_CANDIDIASIS___B37                               0.529697  GEST_DIABETES   \n",
       "AB1_GASTROENTERITIS_NOS___A09                       0.836931  GEST_DIABETES   \n",
       "AB1_OTHER_SEPSIS___A41                              0.729840  GEST_DIABETES   \n",
       "AB1_SEPSIS___A41                                    1.581424  GEST_DIABETES   \n",
       "...                                                      ...            ...   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                0.342948  GEST_DIABETES   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  0.305386  GEST_DIABETES   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  0.433588  GEST_DIABETES   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41       0.389700  GEST_DIABETES   \n",
       "Z21_SPECIAL_SCREEN_EXAM_INFECTIOUS_PARASITIC_DI...  1.087701  GEST_DIABETES   \n",
       "\n",
       "                                                    grep_stat      pval  \\\n",
       "AB1_BACT_INTEST_OTH___A04                            0.969821  0.332136   \n",
       "AB1_CANDIDIASIS___B37                               -0.623860  0.532719   \n",
       "AB1_GASTROENTERITIS_NOS___A09                        1.293344  0.195892   \n",
       "AB1_OTHER_SEPSIS___A41                               0.419651  0.674741   \n",
       "AB1_SEPSIS___A41                                     0.144156  0.885377   \n",
       "...                                                       ...       ...   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                 0.714843  0.474706   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...   2.581369  0.009841   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...   1.531159  0.125730   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41        1.055445  0.291222   \n",
       "Z21_SPECIAL_SCREEN_EXAM_INFECTIOUS_PARASITIC_DI...  -0.066699  0.946821   \n",
       "\n",
       "                                                                                                   p2  \\\n",
       "AB1_BACT_INTEST_OTH___A04                                                   AB1_BACT_INTEST_OTH___A04   \n",
       "AB1_CANDIDIASIS___B37                                                           AB1_CANDIDIASIS___B37   \n",
       "AB1_GASTROENTERITIS_NOS___A09                                           AB1_GASTROENTERITIS_NOS___A09   \n",
       "AB1_OTHER_SEPSIS___A41                                                         AB1_OTHER_SEPSIS___A41   \n",
       "AB1_SEPSIS___A41                                                                     AB1_SEPSIS___A41   \n",
       "...                                                                                               ...   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                             Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA...   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_P...   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41           Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41   \n",
       "Z21_SPECIAL_SCREEN_EXAM_INFECTIOUS_PARASITIC_DI...  Z21_SPECIAL_SCREEN_EXAM_INFECTIOUS_PARASITIC_D...   \n",
       "\n",
       "                                                   significant  \n",
       "AB1_BACT_INTEST_OTH___A04                                       \n",
       "AB1_CANDIDIASIS___B37                                           \n",
       "AB1_GASTROENTERITIS_NOS___A09                                   \n",
       "AB1_OTHER_SEPSIS___A41                                          \n",
       "AB1_SEPSIS___A41                                                \n",
       "...                                                        ...  \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                            \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...              \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...              \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41                   \n",
       "Z21_SPECIAL_SCREEN_EXAM_INFECTIOUS_PARASITIC_DI...              \n",
       "\n",
       "[441 rows x 8 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cor_All</th>\n",
       "      <th>cor</th>\n",
       "      <th>cor_std</th>\n",
       "      <th>trait</th>\n",
       "      <th>grep_stat</th>\n",
       "      <th>pval</th>\n",
       "      <th>p2</th>\n",
       "      <th>significant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AB1_BACT_INTEST_OTH___A04</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>-0.303086</td>\n",
       "      <td>0.332959</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>-0.910280</td>\n",
       "      <td>0.362675</td>\n",
       "      <td>AB1_BACT_INTEST_OTH___A04</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_CANDIDIASIS___B37</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.232431</td>\n",
       "      <td>0.406713</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>0.571487</td>\n",
       "      <td>0.567670</td>\n",
       "      <td>AB1_CANDIDIASIS___B37</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_GASTROENTERITIS_NOS___A09</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.244028</td>\n",
       "      <td>0.282443</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>0.863990</td>\n",
       "      <td>0.387593</td>\n",
       "      <td>AB1_GASTROENTERITIS_NOS___A09</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_OTHER_SEPSIS___A41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.114207</td>\n",
       "      <td>0.325601</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>0.350758</td>\n",
       "      <td>0.725770</td>\n",
       "      <td>AB1_OTHER_SEPSIS___A41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_SEPSIS___A41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.094639</td>\n",
       "      <td>0.364655</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>0.259530</td>\n",
       "      <td>0.795226</td>\n",
       "      <td>AB1_SEPSIS___A41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI___Z03</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.373732</td>\n",
       "      <td>0.194132</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>1.925144</td>\n",
       "      <td>0.054211</td>\n",
       "      <td>Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI_...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.135456</td>\n",
       "      <td>0.201727</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>0.671482</td>\n",
       "      <td>0.501914</td>\n",
       "      <td>Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_REPOR_DIAGNOSIS___Z01</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.361813</td>\n",
       "      <td>0.170499</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>2.122083</td>\n",
       "      <td>0.033831</td>\n",
       "      <td>Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PROCED_NOT_CARRIED___Z53</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.487251</td>\n",
       "      <td>0.360603</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>1.351212</td>\n",
       "      <td>0.176628</td>\n",
       "      <td>Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_P...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.503807</td>\n",
       "      <td>0.279382</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>1.803291</td>\n",
       "      <td>0.071343</td>\n",
       "      <td>Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>447 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Cor_All        cor  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           Cor_All  -0.303086   \n",
       "AB1_CANDIDIASIS___B37                               Cor_All   0.232431   \n",
       "AB1_GASTROENTERITIS_NOS___A09                       Cor_All   0.244028   \n",
       "AB1_OTHER_SEPSIS___A41                              Cor_All   0.114207   \n",
       "AB1_SEPSIS___A41                                    Cor_All   0.094639   \n",
       "...                                                     ...        ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  Cor_All   0.373732   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                Cor_All   0.135456   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  Cor_All   0.361813   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  Cor_All   0.487251   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41       Cor_All   0.503807   \n",
       "\n",
       "                                                     cor_std           trait  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           0.332959  I9_HYPTENSPREG   \n",
       "AB1_CANDIDIASIS___B37                               0.406713  I9_HYPTENSPREG   \n",
       "AB1_GASTROENTERITIS_NOS___A09                       0.282443  I9_HYPTENSPREG   \n",
       "AB1_OTHER_SEPSIS___A41                              0.325601  I9_HYPTENSPREG   \n",
       "AB1_SEPSIS___A41                                    0.364655  I9_HYPTENSPREG   \n",
       "...                                                      ...             ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  0.194132  I9_HYPTENSPREG   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                0.201727  I9_HYPTENSPREG   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  0.170499  I9_HYPTENSPREG   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  0.360603  I9_HYPTENSPREG   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41       0.279382  I9_HYPTENSPREG   \n",
       "\n",
       "                                                    grep_stat      pval  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           -0.910280  0.362675   \n",
       "AB1_CANDIDIASIS___B37                                0.571487  0.567670   \n",
       "AB1_GASTROENTERITIS_NOS___A09                        0.863990  0.387593   \n",
       "AB1_OTHER_SEPSIS___A41                               0.350758  0.725770   \n",
       "AB1_SEPSIS___A41                                     0.259530  0.795226   \n",
       "...                                                       ...       ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...   1.925144  0.054211   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                 0.671482  0.501914   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...   2.122083  0.033831   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...   1.351212  0.176628   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41        1.803291  0.071343   \n",
       "\n",
       "                                                                                                   p2  \\\n",
       "AB1_BACT_INTEST_OTH___A04                                                   AB1_BACT_INTEST_OTH___A04   \n",
       "AB1_CANDIDIASIS___B37                                                           AB1_CANDIDIASIS___B37   \n",
       "AB1_GASTROENTERITIS_NOS___A09                                           AB1_GASTROENTERITIS_NOS___A09   \n",
       "AB1_OTHER_SEPSIS___A41                                                         AB1_OTHER_SEPSIS___A41   \n",
       "AB1_SEPSIS___A41                                                                     AB1_SEPSIS___A41   \n",
       "...                                                                                               ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI_...   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                             Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA...   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_P...   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41           Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41   \n",
       "\n",
       "                                                   significant  \n",
       "AB1_BACT_INTEST_OTH___A04                                       \n",
       "AB1_CANDIDIASIS___B37                                           \n",
       "AB1_GASTROENTERITIS_NOS___A09                                   \n",
       "AB1_OTHER_SEPSIS___A41                                          \n",
       "AB1_SEPSIS___A41                                                \n",
       "...                                                        ...  \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...              \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                            \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...              \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...              \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41                   \n",
       "\n",
       "[447 rows x 8 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cor_All</th>\n",
       "      <th>cor</th>\n",
       "      <th>cor_std</th>\n",
       "      <th>trait</th>\n",
       "      <th>grep_stat</th>\n",
       "      <th>pval</th>\n",
       "      <th>p2</th>\n",
       "      <th>significant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AB1_BACT_INTEST_OTH___A04</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>-0.271273</td>\n",
       "      <td>0.556522</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>-0.487443</td>\n",
       "      <td>0.625944</td>\n",
       "      <td>AB1_BACT_INTEST_OTH___A04</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_CANDIDIASIS___B37</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.500804</td>\n",
       "      <td>0.634897</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.788796</td>\n",
       "      <td>0.430231</td>\n",
       "      <td>AB1_CANDIDIASIS___B37</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_GASTROENTERITIS_NOS___A09</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.354631</td>\n",
       "      <td>0.561824</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.631214</td>\n",
       "      <td>0.527901</td>\n",
       "      <td>AB1_GASTROENTERITIS_NOS___A09</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_OTHER_SEPSIS___A41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.484202</td>\n",
       "      <td>0.642383</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.753759</td>\n",
       "      <td>0.450994</td>\n",
       "      <td>AB1_OTHER_SEPSIS___A41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_SEPSIS___A41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.457686</td>\n",
       "      <td>0.738186</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.620014</td>\n",
       "      <td>0.535248</td>\n",
       "      <td>AB1_SEPSIS___A41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI___Z03</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>1.102863</td>\n",
       "      <td>0.678434</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>1.625601</td>\n",
       "      <td>0.104035</td>\n",
       "      <td>Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI_...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.255791</td>\n",
       "      <td>0.476292</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.537047</td>\n",
       "      <td>0.591235</td>\n",
       "      <td>Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_REPOR_DIAGNOSIS___Z01</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.749449</td>\n",
       "      <td>0.500880</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>1.496265</td>\n",
       "      <td>0.134585</td>\n",
       "      <td>Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PROCED_NOT_CARRIED___Z53</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>-0.213836</td>\n",
       "      <td>0.617483</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>-0.346303</td>\n",
       "      <td>0.729115</td>\n",
       "      <td>Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_P...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.187718</td>\n",
       "      <td>0.558837</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.335908</td>\n",
       "      <td>0.736940</td>\n",
       "      <td>Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>443 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Cor_All        cor  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           Cor_All  -0.271273   \n",
       "AB1_CANDIDIASIS___B37                               Cor_All   0.500804   \n",
       "AB1_GASTROENTERITIS_NOS___A09                       Cor_All   0.354631   \n",
       "AB1_OTHER_SEPSIS___A41                              Cor_All   0.484202   \n",
       "AB1_SEPSIS___A41                                    Cor_All   0.457686   \n",
       "...                                                     ...        ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  Cor_All   1.102863   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                Cor_All   0.255791   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  Cor_All   0.749449   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  Cor_All  -0.213836   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41       Cor_All   0.187718   \n",
       "\n",
       "                                                     cor_std        trait  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           0.556522  O15_PRETERM   \n",
       "AB1_CANDIDIASIS___B37                               0.634897  O15_PRETERM   \n",
       "AB1_GASTROENTERITIS_NOS___A09                       0.561824  O15_PRETERM   \n",
       "AB1_OTHER_SEPSIS___A41                              0.642383  O15_PRETERM   \n",
       "AB1_SEPSIS___A41                                    0.738186  O15_PRETERM   \n",
       "...                                                      ...          ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  0.678434  O15_PRETERM   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                0.476292  O15_PRETERM   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  0.500880  O15_PRETERM   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  0.617483  O15_PRETERM   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41       0.558837  O15_PRETERM   \n",
       "\n",
       "                                                    grep_stat      pval  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           -0.487443  0.625944   \n",
       "AB1_CANDIDIASIS___B37                                0.788796  0.430231   \n",
       "AB1_GASTROENTERITIS_NOS___A09                        0.631214  0.527901   \n",
       "AB1_OTHER_SEPSIS___A41                               0.753759  0.450994   \n",
       "AB1_SEPSIS___A41                                     0.620014  0.535248   \n",
       "...                                                       ...       ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...   1.625601  0.104035   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                 0.537047  0.591235   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...   1.496265  0.134585   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  -0.346303  0.729115   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41        0.335908  0.736940   \n",
       "\n",
       "                                                                                                   p2  \\\n",
       "AB1_BACT_INTEST_OTH___A04                                                   AB1_BACT_INTEST_OTH___A04   \n",
       "AB1_CANDIDIASIS___B37                                                           AB1_CANDIDIASIS___B37   \n",
       "AB1_GASTROENTERITIS_NOS___A09                                           AB1_GASTROENTERITIS_NOS___A09   \n",
       "AB1_OTHER_SEPSIS___A41                                                         AB1_OTHER_SEPSIS___A41   \n",
       "AB1_SEPSIS___A41                                                                     AB1_SEPSIS___A41   \n",
       "...                                                                                               ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI_...   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                             Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA...   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_P...   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41           Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41   \n",
       "\n",
       "                                                   significant  \n",
       "AB1_BACT_INTEST_OTH___A04                                       \n",
       "AB1_CANDIDIASIS___B37                                           \n",
       "AB1_GASTROENTERITIS_NOS___A09                                   \n",
       "AB1_OTHER_SEPSIS___A41                                          \n",
       "AB1_SEPSIS___A41                                                \n",
       "...                                                        ...  \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...              \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                            \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...              \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...              \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41                   \n",
       "\n",
       "[443 rows x 8 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gest_data  = return_df(GEST)\n",
    "hypt_data = return_df(HYPT)\n",
    "pret_data = return_df(PRET)\n",
    "gest_data  = count_sign(gest_data)\n",
    "hypt_data = count_sign(hypt_data)\n",
    "pret_data = count_sign(pret_data)\n",
    "display(gest_data)\n",
    "display(hypt_data)\n",
    "display(pret_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f69b06a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cor_All</th>\n",
       "      <th>cor</th>\n",
       "      <th>cor_std</th>\n",
       "      <th>trait</th>\n",
       "      <th>grep_stat</th>\n",
       "      <th>pval</th>\n",
       "      <th>p2</th>\n",
       "      <th>significant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AB1_BACT_INTEST_OTH___A04</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.434338</td>\n",
       "      <td>0.447854</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>0.969821</td>\n",
       "      <td>0.332136</td>\n",
       "      <td>AB1_BACT_INTEST_OTH___A04</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_CANDIDIASIS___B37</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>-0.330457</td>\n",
       "      <td>0.529697</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>-0.623860</td>\n",
       "      <td>0.532719</td>\n",
       "      <td>AB1_CANDIDIASIS___B37</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_GASTROENTERITIS_NOS___A09</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>1.082440</td>\n",
       "      <td>0.836931</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>1.293344</td>\n",
       "      <td>0.195892</td>\n",
       "      <td>AB1_GASTROENTERITIS_NOS___A09</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_OTHER_SEPSIS___A41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.306278</td>\n",
       "      <td>0.729840</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>0.419651</td>\n",
       "      <td>0.674741</td>\n",
       "      <td>AB1_OTHER_SEPSIS___A41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AB1_SEPSIS___A41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.227972</td>\n",
       "      <td>1.581424</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>0.144156</td>\n",
       "      <td>0.885377</td>\n",
       "      <td>AB1_SEPSIS___A41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI___Z03</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>1.102863</td>\n",
       "      <td>0.678434</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>1.625601</td>\n",
       "      <td>0.104035</td>\n",
       "      <td>Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI_...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.255791</td>\n",
       "      <td>0.476292</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.537047</td>\n",
       "      <td>0.591235</td>\n",
       "      <td>Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_REPOR_DIAGNOSIS___Z01</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.749449</td>\n",
       "      <td>0.500880</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>1.496265</td>\n",
       "      <td>0.134585</td>\n",
       "      <td>Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PROCED_NOT_CARRIED___Z53</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>-0.213836</td>\n",
       "      <td>0.617483</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>-0.346303</td>\n",
       "      <td>0.729115</td>\n",
       "      <td>Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_P...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.187718</td>\n",
       "      <td>0.558837</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.335908</td>\n",
       "      <td>0.736940</td>\n",
       "      <td>Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1331 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    Cor_All        cor  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           Cor_All   0.434338   \n",
       "AB1_CANDIDIASIS___B37                               Cor_All  -0.330457   \n",
       "AB1_GASTROENTERITIS_NOS___A09                       Cor_All   1.082440   \n",
       "AB1_OTHER_SEPSIS___A41                              Cor_All   0.306278   \n",
       "AB1_SEPSIS___A41                                    Cor_All   0.227972   \n",
       "...                                                     ...        ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  Cor_All   1.102863   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                Cor_All   0.255791   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  Cor_All   0.749449   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  Cor_All  -0.213836   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41       Cor_All   0.187718   \n",
       "\n",
       "                                                     cor_std          trait  \\\n",
       "AB1_BACT_INTEST_OTH___A04                           0.447854  GEST_DIABETES   \n",
       "AB1_CANDIDIASIS___B37                               0.529697  GEST_DIABETES   \n",
       "AB1_GASTROENTERITIS_NOS___A09                       0.836931  GEST_DIABETES   \n",
       "AB1_OTHER_SEPSIS___A41                              0.729840  GEST_DIABETES   \n",
       "AB1_SEPSIS___A41                                    1.581424  GEST_DIABETES   \n",
       "...                                                      ...            ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  0.678434    O15_PRETERM   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                0.476292    O15_PRETERM   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  0.500880    O15_PRETERM   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  0.617483    O15_PRETERM   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41       0.558837    O15_PRETERM   \n",
       "\n",
       "                                                    grep_stat      pval  \\\n",
       "AB1_BACT_INTEST_OTH___A04                            0.969821  0.332136   \n",
       "AB1_CANDIDIASIS___B37                               -0.623860  0.532719   \n",
       "AB1_GASTROENTERITIS_NOS___A09                        1.293344  0.195892   \n",
       "AB1_OTHER_SEPSIS___A41                               0.419651  0.674741   \n",
       "AB1_SEPSIS___A41                                     0.144156  0.885377   \n",
       "...                                                       ...       ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...   1.625601  0.104035   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                 0.537047  0.591235   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...   1.496265  0.134585   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  -0.346303  0.729115   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41        0.335908  0.736940   \n",
       "\n",
       "                                                                                                   p2  \\\n",
       "AB1_BACT_INTEST_OTH___A04                                                   AB1_BACT_INTEST_OTH___A04   \n",
       "AB1_CANDIDIASIS___B37                                                           AB1_CANDIDIASIS___B37   \n",
       "AB1_GASTROENTERITIS_NOS___A09                                           AB1_GASTROENTERITIS_NOS___A09   \n",
       "AB1_OTHER_SEPSIS___A41                                                         AB1_OTHER_SEPSIS___A41   \n",
       "AB1_SEPSIS___A41                                                                     AB1_SEPSIS___A41   \n",
       "...                                                                                               ...   \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...  Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI_...   \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                             Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47   \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...  Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA...   \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...  Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_P...   \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41           Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41   \n",
       "\n",
       "                                                   significant  \n",
       "AB1_BACT_INTEST_OTH___A04                                       \n",
       "AB1_CANDIDIASIS___B37                                           \n",
       "AB1_GASTROENTERITIS_NOS___A09                                   \n",
       "AB1_OTHER_SEPSIS___A41                                          \n",
       "AB1_SEPSIS___A41                                                \n",
       "...                                                        ...  \n",
       "Z21_MEDICAL_OBSER_EVALU_SUSPECTED_DISEA_CONDI__...              \n",
       "Z21_OTHER_ORTHOPA_FOLLOW__CARE___Z47                            \n",
       "Z21_OTHER_SPECIAL_EXAM_INVES_PERSONS_WO_COMPLA_...              \n",
       "Z21_PERSONS_ENCOUNTERI_HEALTH_SERVI_SPECIFIC_PR...              \n",
       "Z21_PROCED_PURPO_OTH_REMED_HEALTH_STATE___Z41                   \n",
       "\n",
       "[1331 rows x 8 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concatted_data = pd.concat([gest_data, hypt_data, pret_data])\n",
    "concatted_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0c94ee03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cor_All</th>\n",
       "      <th>cor</th>\n",
       "      <th>cor_std</th>\n",
       "      <th>trait</th>\n",
       "      <th>grep_stat</th>\n",
       "      <th>pval</th>\n",
       "      <th>p2</th>\n",
       "      <th>significant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.930488</td>\n",
       "      <td>0.164872</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>5.643699</td>\n",
       "      <td>1.664346e-08</td>\n",
       "      <td>E4_DIABETES___E14</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.529482</td>\n",
       "      <td>0.153193</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>3.456307</td>\n",
       "      <td>5.476317e-04</td>\n",
       "      <td>E4_OBESITY___E66</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.500904</td>\n",
       "      <td>0.141347</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>3.543789</td>\n",
       "      <td>3.944202e-04</td>\n",
       "      <td>I9_HYPTENSESS___I10</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.247124</td>\n",
       "      <td>0.091702</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>2.694859</td>\n",
       "      <td>7.041833e-03</td>\n",
       "      <td>I9_IHD___I25</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.107861</td>\n",
       "      <td>0.474805</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>0.227169</td>\n",
       "      <td>8.202923e-01</td>\n",
       "      <td>O15_MATERN_HYPERT_UNS___O16</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_PRETERM___O60</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>-0.065977</td>\n",
       "      <td>0.465984</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>-0.141586</td>\n",
       "      <td>8.874067e-01</td>\n",
       "      <td>O15_PRETERM___O60</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.410649</td>\n",
       "      <td>0.114888</td>\n",
       "      <td>GEST_DIABETES</td>\n",
       "      <td>3.574342</td>\n",
       "      <td>3.511098e-04</td>\n",
       "      <td>R18_PAIN_THROAT_CHEST___R07</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.420760</td>\n",
       "      <td>0.098911</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>4.253925</td>\n",
       "      <td>2.100555e-05</td>\n",
       "      <td>E4_DIABETES___E14</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.374283</td>\n",
       "      <td>0.092909</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>4.028490</td>\n",
       "      <td>5.613619e-05</td>\n",
       "      <td>E4_OBESITY___E66</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.699877</td>\n",
       "      <td>0.101525</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>6.893642</td>\n",
       "      <td>5.438180e-12</td>\n",
       "      <td>I9_HYPTENSESS___I10</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.450153</td>\n",
       "      <td>0.083383</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>5.398618</td>\n",
       "      <td>6.715603e-08</td>\n",
       "      <td>I9_IHD___I25</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.781216</td>\n",
       "      <td>0.178088</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>4.386685</td>\n",
       "      <td>1.150912e-05</td>\n",
       "      <td>O15_MATERN_HYPERT_UNS___O16</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_PRETERM___O60</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.466229</td>\n",
       "      <td>0.476958</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>0.977505</td>\n",
       "      <td>3.283190e-01</td>\n",
       "      <td>O15_PRETERM___O60</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.358103</td>\n",
       "      <td>0.078627</td>\n",
       "      <td>I9_HYPTENSPREG</td>\n",
       "      <td>4.554453</td>\n",
       "      <td>5.252199e-06</td>\n",
       "      <td>R18_PAIN_THROAT_CHEST___R07</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.300049</td>\n",
       "      <td>0.316877</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.946894</td>\n",
       "      <td>3.436927e-01</td>\n",
       "      <td>E4_DIABETES___E14</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.175672</td>\n",
       "      <td>0.260471</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.674440</td>\n",
       "      <td>5.000318e-01</td>\n",
       "      <td>E4_OBESITY___E66</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.319135</td>\n",
       "      <td>0.311526</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>1.024425</td>\n",
       "      <td>3.056346e-01</td>\n",
       "      <td>I9_HYPTENSESS___I10</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.148780</td>\n",
       "      <td>0.158187</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.940532</td>\n",
       "      <td>3.469445e-01</td>\n",
       "      <td>I9_IHD___I25</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.089876</td>\n",
       "      <td>0.615927</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>0.145920</td>\n",
       "      <td>8.839846e-01</td>\n",
       "      <td>O15_MATERN_HYPERT_UNS___O16</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_PRETERM___O60</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.979330</td>\n",
       "      <td>0.021307</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>45.962829</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>O15_PRETERM___O60</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.321487</td>\n",
       "      <td>0.220537</td>\n",
       "      <td>O15_PRETERM</td>\n",
       "      <td>1.457746</td>\n",
       "      <td>1.449105e-01</td>\n",
       "      <td>R18_PAIN_THROAT_CHEST___R07</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             Cor_All        cor   cor_std           trait  \\\n",
       "E4_DIABETES___E14            Cor_All   0.930488  0.164872   GEST_DIABETES   \n",
       "E4_OBESITY___E66             Cor_All   0.529482  0.153193   GEST_DIABETES   \n",
       "I9_HYPTENSESS___I10          Cor_All   0.500904  0.141347   GEST_DIABETES   \n",
       "I9_IHD___I25                 Cor_All   0.247124  0.091702   GEST_DIABETES   \n",
       "O15_MATERN_HYPERT_UNS___O16  Cor_All   0.107861  0.474805   GEST_DIABETES   \n",
       "O15_PRETERM___O60            Cor_All  -0.065977  0.465984   GEST_DIABETES   \n",
       "R18_PAIN_THROAT_CHEST___R07  Cor_All   0.410649  0.114888   GEST_DIABETES   \n",
       "E4_DIABETES___E14            Cor_All   0.420760  0.098911  I9_HYPTENSPREG   \n",
       "E4_OBESITY___E66             Cor_All   0.374283  0.092909  I9_HYPTENSPREG   \n",
       "I9_HYPTENSESS___I10          Cor_All   0.699877  0.101525  I9_HYPTENSPREG   \n",
       "I9_IHD___I25                 Cor_All   0.450153  0.083383  I9_HYPTENSPREG   \n",
       "O15_MATERN_HYPERT_UNS___O16  Cor_All   0.781216  0.178088  I9_HYPTENSPREG   \n",
       "O15_PRETERM___O60            Cor_All   0.466229  0.476958  I9_HYPTENSPREG   \n",
       "R18_PAIN_THROAT_CHEST___R07  Cor_All   0.358103  0.078627  I9_HYPTENSPREG   \n",
       "E4_DIABETES___E14            Cor_All   0.300049  0.316877     O15_PRETERM   \n",
       "E4_OBESITY___E66             Cor_All   0.175672  0.260471     O15_PRETERM   \n",
       "I9_HYPTENSESS___I10          Cor_All   0.319135  0.311526     O15_PRETERM   \n",
       "I9_IHD___I25                 Cor_All   0.148780  0.158187     O15_PRETERM   \n",
       "O15_MATERN_HYPERT_UNS___O16  Cor_All   0.089876  0.615927     O15_PRETERM   \n",
       "O15_PRETERM___O60            Cor_All   0.979330  0.021307     O15_PRETERM   \n",
       "R18_PAIN_THROAT_CHEST___R07  Cor_All   0.321487  0.220537     O15_PRETERM   \n",
       "\n",
       "                             grep_stat          pval  \\\n",
       "E4_DIABETES___E14             5.643699  1.664346e-08   \n",
       "E4_OBESITY___E66              3.456307  5.476317e-04   \n",
       "I9_HYPTENSESS___I10           3.543789  3.944202e-04   \n",
       "I9_IHD___I25                  2.694859  7.041833e-03   \n",
       "O15_MATERN_HYPERT_UNS___O16   0.227169  8.202923e-01   \n",
       "O15_PRETERM___O60            -0.141586  8.874067e-01   \n",
       "R18_PAIN_THROAT_CHEST___R07   3.574342  3.511098e-04   \n",
       "E4_DIABETES___E14             4.253925  2.100555e-05   \n",
       "E4_OBESITY___E66              4.028490  5.613619e-05   \n",
       "I9_HYPTENSESS___I10           6.893642  5.438180e-12   \n",
       "I9_IHD___I25                  5.398618  6.715603e-08   \n",
       "O15_MATERN_HYPERT_UNS___O16   4.386685  1.150912e-05   \n",
       "O15_PRETERM___O60             0.977505  3.283190e-01   \n",
       "R18_PAIN_THROAT_CHEST___R07   4.554453  5.252199e-06   \n",
       "E4_DIABETES___E14             0.946894  3.436927e-01   \n",
       "E4_OBESITY___E66              0.674440  5.000318e-01   \n",
       "I9_HYPTENSESS___I10           1.024425  3.056346e-01   \n",
       "I9_IHD___I25                  0.940532  3.469445e-01   \n",
       "O15_MATERN_HYPERT_UNS___O16   0.145920  8.839846e-01   \n",
       "O15_PRETERM___O60            45.962829  0.000000e+00   \n",
       "R18_PAIN_THROAT_CHEST___R07   1.457746  1.449105e-01   \n",
       "\n",
       "                                                      p2 significant  \n",
       "E4_DIABETES___E14                      E4_DIABETES___E14           *  \n",
       "E4_OBESITY___E66                        E4_OBESITY___E66              \n",
       "I9_HYPTENSESS___I10                  I9_HYPTENSESS___I10              \n",
       "I9_IHD___I25                                I9_IHD___I25              \n",
       "O15_MATERN_HYPERT_UNS___O16  O15_MATERN_HYPERT_UNS___O16              \n",
       "O15_PRETERM___O60                      O15_PRETERM___O60              \n",
       "R18_PAIN_THROAT_CHEST___R07  R18_PAIN_THROAT_CHEST___R07              \n",
       "E4_DIABETES___E14                      E4_DIABETES___E14           *  \n",
       "E4_OBESITY___E66                        E4_OBESITY___E66           *  \n",
       "I9_HYPTENSESS___I10                  I9_HYPTENSESS___I10           *  \n",
       "I9_IHD___I25                                I9_IHD___I25           *  \n",
       "O15_MATERN_HYPERT_UNS___O16  O15_MATERN_HYPERT_UNS___O16           *  \n",
       "O15_PRETERM___O60                      O15_PRETERM___O60              \n",
       "R18_PAIN_THROAT_CHEST___R07  R18_PAIN_THROAT_CHEST___R07           *  \n",
       "E4_DIABETES___E14                      E4_DIABETES___E14              \n",
       "E4_OBESITY___E66                        E4_OBESITY___E66              \n",
       "I9_HYPTENSESS___I10                  I9_HYPTENSESS___I10              \n",
       "I9_IHD___I25                                I9_IHD___I25              \n",
       "O15_MATERN_HYPERT_UNS___O16  O15_MATERN_HYPERT_UNS___O16              \n",
       "O15_PRETERM___O60                      O15_PRETERM___O60           *  \n",
       "R18_PAIN_THROAT_CHEST___R07  R18_PAIN_THROAT_CHEST___R07              "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_concatted_data = concatted_data[concatted_data.significant=='*']\n",
    "cd_final = concatted_data[concatted_data.p2.isin(set(filtered_concatted_data.index))]\n",
    "cd_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ac81c784",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cor_All</th>\n",
       "      <th>rg</th>\n",
       "      <th>se</th>\n",
       "      <th>p1</th>\n",
       "      <th>z</th>\n",
       "      <th>p</th>\n",
       "      <th>p2</th>\n",
       "      <th>significant</th>\n",
       "      <th>full_name</th>\n",
       "      <th>priority</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.930488</td>\n",
       "      <td>0.164872</td>\n",
       "      <td>GD</td>\n",
       "      <td>5.643699</td>\n",
       "      <td>1.664346e-08</td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td>*</td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.420760</td>\n",
       "      <td>0.098911</td>\n",
       "      <td>HP</td>\n",
       "      <td>4.253925</td>\n",
       "      <td>2.100555e-05</td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td>*</td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.300049</td>\n",
       "      <td>0.316877</td>\n",
       "      <td>PTB</td>\n",
       "      <td>0.946894</td>\n",
       "      <td>3.436927e-01</td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.529482</td>\n",
       "      <td>0.153193</td>\n",
       "      <td>GD</td>\n",
       "      <td>3.456307</td>\n",
       "      <td>5.476317e-04</td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td></td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.374283</td>\n",
       "      <td>0.092909</td>\n",
       "      <td>HP</td>\n",
       "      <td>4.028490</td>\n",
       "      <td>5.613619e-05</td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td>*</td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.175672</td>\n",
       "      <td>0.260471</td>\n",
       "      <td>PTB</td>\n",
       "      <td>0.674440</td>\n",
       "      <td>5.000318e-01</td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td></td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.500904</td>\n",
       "      <td>0.141347</td>\n",
       "      <td>GD</td>\n",
       "      <td>3.543789</td>\n",
       "      <td>3.944202e-04</td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.699877</td>\n",
       "      <td>0.101525</td>\n",
       "      <td>HP</td>\n",
       "      <td>6.893642</td>\n",
       "      <td>5.438180e-12</td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td>*</td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.319135</td>\n",
       "      <td>0.311526</td>\n",
       "      <td>PTB</td>\n",
       "      <td>1.024425</td>\n",
       "      <td>3.056346e-01</td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.247124</td>\n",
       "      <td>0.091702</td>\n",
       "      <td>GD</td>\n",
       "      <td>2.694859</td>\n",
       "      <td>7.041833e-03</td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.450153</td>\n",
       "      <td>0.083383</td>\n",
       "      <td>HP</td>\n",
       "      <td>5.398618</td>\n",
       "      <td>6.715603e-08</td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td>*</td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.148780</td>\n",
       "      <td>0.158187</td>\n",
       "      <td>PTB</td>\n",
       "      <td>0.940532</td>\n",
       "      <td>3.469445e-01</td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.107861</td>\n",
       "      <td>0.474805</td>\n",
       "      <td>GD</td>\n",
       "      <td>0.227169</td>\n",
       "      <td>8.202923e-01</td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.781216</td>\n",
       "      <td>0.178088</td>\n",
       "      <td>HP</td>\n",
       "      <td>4.386685</td>\n",
       "      <td>1.150912e-05</td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td>*</td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.089876</td>\n",
       "      <td>0.615927</td>\n",
       "      <td>PTB</td>\n",
       "      <td>0.145920</td>\n",
       "      <td>8.839846e-01</td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.410649</td>\n",
       "      <td>0.114888</td>\n",
       "      <td>GD</td>\n",
       "      <td>3.574342</td>\n",
       "      <td>3.511098e-04</td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.358103</td>\n",
       "      <td>0.078627</td>\n",
       "      <td>HP</td>\n",
       "      <td>4.554453</td>\n",
       "      <td>5.252199e-06</td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td>*</td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>Cor_All</td>\n",
       "      <td>0.321487</td>\n",
       "      <td>0.220537</td>\n",
       "      <td>PTB</td>\n",
       "      <td>1.457746</td>\n",
       "      <td>1.449105e-01</td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td></td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             Cor_All        rg        se   p1         z  \\\n",
       "E4_DIABETES___E14            Cor_All  0.930488  0.164872   GD  5.643699   \n",
       "E4_DIABETES___E14            Cor_All  0.420760  0.098911   HP  4.253925   \n",
       "E4_DIABETES___E14            Cor_All  0.300049  0.316877  PTB  0.946894   \n",
       "E4_OBESITY___E66             Cor_All  0.529482  0.153193   GD  3.456307   \n",
       "E4_OBESITY___E66             Cor_All  0.374283  0.092909   HP  4.028490   \n",
       "E4_OBESITY___E66             Cor_All  0.175672  0.260471  PTB  0.674440   \n",
       "I9_HYPTENSESS___I10          Cor_All  0.500904  0.141347   GD  3.543789   \n",
       "I9_HYPTENSESS___I10          Cor_All  0.699877  0.101525   HP  6.893642   \n",
       "I9_HYPTENSESS___I10          Cor_All  0.319135  0.311526  PTB  1.024425   \n",
       "I9_IHD___I25                 Cor_All  0.247124  0.091702   GD  2.694859   \n",
       "I9_IHD___I25                 Cor_All  0.450153  0.083383   HP  5.398618   \n",
       "I9_IHD___I25                 Cor_All  0.148780  0.158187  PTB  0.940532   \n",
       "O15_MATERN_HYPERT_UNS___O16  Cor_All  0.107861  0.474805   GD  0.227169   \n",
       "O15_MATERN_HYPERT_UNS___O16  Cor_All  0.781216  0.178088   HP  4.386685   \n",
       "O15_MATERN_HYPERT_UNS___O16  Cor_All  0.089876  0.615927  PTB  0.145920   \n",
       "R18_PAIN_THROAT_CHEST___R07  Cor_All  0.410649  0.114888   GD  3.574342   \n",
       "R18_PAIN_THROAT_CHEST___R07  Cor_All  0.358103  0.078627   HP  4.554453   \n",
       "R18_PAIN_THROAT_CHEST___R07  Cor_All  0.321487  0.220537  PTB  1.457746   \n",
       "\n",
       "                                        p  \\\n",
       "E4_DIABETES___E14            1.664346e-08   \n",
       "E4_DIABETES___E14            2.100555e-05   \n",
       "E4_DIABETES___E14            3.436927e-01   \n",
       "E4_OBESITY___E66             5.476317e-04   \n",
       "E4_OBESITY___E66             5.613619e-05   \n",
       "E4_OBESITY___E66             5.000318e-01   \n",
       "I9_HYPTENSESS___I10          3.944202e-04   \n",
       "I9_HYPTENSESS___I10          5.438180e-12   \n",
       "I9_HYPTENSESS___I10          3.056346e-01   \n",
       "I9_IHD___I25                 7.041833e-03   \n",
       "I9_IHD___I25                 6.715603e-08   \n",
       "I9_IHD___I25                 3.469445e-01   \n",
       "O15_MATERN_HYPERT_UNS___O16  8.202923e-01   \n",
       "O15_MATERN_HYPERT_UNS___O16  1.150912e-05   \n",
       "O15_MATERN_HYPERT_UNS___O16  8.839846e-01   \n",
       "R18_PAIN_THROAT_CHEST___R07  3.511098e-04   \n",
       "R18_PAIN_THROAT_CHEST___R07  5.252199e-06   \n",
       "R18_PAIN_THROAT_CHEST___R07  1.449105e-01   \n",
       "\n",
       "                                                                            p2  \\\n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "\n",
       "                            significant  \\\n",
       "E4_DIABETES___E14                     *   \n",
       "E4_DIABETES___E14                     *   \n",
       "E4_DIABETES___E14                         \n",
       "E4_OBESITY___E66                          \n",
       "E4_OBESITY___E66                      *   \n",
       "E4_OBESITY___E66                          \n",
       "I9_HYPTENSESS___I10                       \n",
       "I9_HYPTENSESS___I10                   *   \n",
       "I9_HYPTENSESS___I10                       \n",
       "I9_IHD___I25                              \n",
       "I9_IHD___I25                          *   \n",
       "I9_IHD___I25                              \n",
       "O15_MATERN_HYPERT_UNS___O16               \n",
       "O15_MATERN_HYPERT_UNS___O16           *   \n",
       "O15_MATERN_HYPERT_UNS___O16               \n",
       "R18_PAIN_THROAT_CHEST___R07               \n",
       "R18_PAIN_THROAT_CHEST___R07           *   \n",
       "R18_PAIN_THROAT_CHEST___R07               \n",
       "\n",
       "                                                                     full_name  \\\n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "\n",
       "                            priority  \n",
       "E4_DIABETES___E14                  2  \n",
       "E4_DIABETES___E14                  2  \n",
       "E4_DIABETES___E14                  2  \n",
       "E4_OBESITY___E66                   2  \n",
       "E4_OBESITY___E66                   2  \n",
       "E4_OBESITY___E66                   2  \n",
       "I9_HYPTENSESS___I10                2  \n",
       "I9_HYPTENSESS___I10                2  \n",
       "I9_HYPTENSESS___I10                2  \n",
       "I9_IHD___I25                       2  \n",
       "I9_IHD___I25                       2  \n",
       "I9_IHD___I25                       2  \n",
       "O15_MATERN_HYPERT_UNS___O16        2  \n",
       "O15_MATERN_HYPERT_UNS___O16        2  \n",
       "O15_MATERN_HYPERT_UNS___O16        2  \n",
       "R18_PAIN_THROAT_CHEST___R07        2  \n",
       "R18_PAIN_THROAT_CHEST___R07        2  \n",
       "R18_PAIN_THROAT_CHEST___R07        2  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final = pd.merge(cd_final, fu,  how=\"left\", left_index=True, right_index=True)\n",
    "final.loc[final.p2== 'I9_HYPTENSESS___I10', ['priority']]='2'\n",
    "final = final[final.index!='O15_PRETERM___O60']\n",
    "\n",
    "col_rename = {\"cor\": \"rg\", \"cor_std\": \"se\", \"trait\": \"p1\", \"grep_stat\": \"z\", \"pval\": \"p\", \"p2\": \"p2\", \"priority\":\"priority\"}\n",
    "final = final.rename(columns=col_rename)\n",
    "\n",
    "final.p1 = final.p1.apply(lambda x: ph_map[x])\n",
    "final.p2 = final.full_name\n",
    "\n",
    "\n",
    "print(final.shape)\n",
    "final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5d25df94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p1</th>\n",
       "      <th>p2</th>\n",
       "      <th>rg</th>\n",
       "      <th>se</th>\n",
       "      <th>z</th>\n",
       "      <th>p</th>\n",
       "      <th>priority</th>\n",
       "      <th>significant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>GD</td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td>0.930488</td>\n",
       "      <td>0.164872</td>\n",
       "      <td>5.643699</td>\n",
       "      <td>1.664346e-08</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>HP</td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td>0.420760</td>\n",
       "      <td>0.098911</td>\n",
       "      <td>4.253925</td>\n",
       "      <td>2.100555e-05</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_DIABETES___E14</th>\n",
       "      <td>PTB</td>\n",
       "      <td>FG: Diabetes mellitus &amp; \\nUK: Unspecified diab...</td>\n",
       "      <td>0.300049</td>\n",
       "      <td>0.316877</td>\n",
       "      <td>0.946894</td>\n",
       "      <td>3.436927e-01</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>GD</td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td>0.529482</td>\n",
       "      <td>0.153193</td>\n",
       "      <td>3.456307</td>\n",
       "      <td>5.476317e-04</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>HP</td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td>0.374283</td>\n",
       "      <td>0.092909</td>\n",
       "      <td>4.028490</td>\n",
       "      <td>5.613619e-05</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>E4_OBESITY___E66</th>\n",
       "      <td>PTB</td>\n",
       "      <td>FG: Obesity &amp; \\nUK: Obesity</td>\n",
       "      <td>0.175672</td>\n",
       "      <td>0.260471</td>\n",
       "      <td>0.674440</td>\n",
       "      <td>5.000318e-01</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>GD</td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td>0.500904</td>\n",
       "      <td>0.141347</td>\n",
       "      <td>3.543789</td>\n",
       "      <td>3.944202e-04</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>HP</td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td>0.699877</td>\n",
       "      <td>0.101525</td>\n",
       "      <td>6.893642</td>\n",
       "      <td>5.438180e-12</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_HYPTENSESS___I10</th>\n",
       "      <td>PTB</td>\n",
       "      <td>FG: Hypertension, essential &amp; \\nUK: Essential ...</td>\n",
       "      <td>0.319135</td>\n",
       "      <td>0.311526</td>\n",
       "      <td>1.024425</td>\n",
       "      <td>3.056346e-01</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>GD</td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td>0.247124</td>\n",
       "      <td>0.091702</td>\n",
       "      <td>2.694859</td>\n",
       "      <td>7.041833e-03</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>HP</td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td>0.450153</td>\n",
       "      <td>0.083383</td>\n",
       "      <td>5.398618</td>\n",
       "      <td>6.715603e-08</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>I9_IHD___I25</th>\n",
       "      <td>PTB</td>\n",
       "      <td>FG: Ischaemic heart disease, wide definition &amp;...</td>\n",
       "      <td>0.148780</td>\n",
       "      <td>0.158187</td>\n",
       "      <td>0.940532</td>\n",
       "      <td>3.469445e-01</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>GD</td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td>0.107861</td>\n",
       "      <td>0.474805</td>\n",
       "      <td>0.227169</td>\n",
       "      <td>8.202923e-01</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>HP</td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td>0.781216</td>\n",
       "      <td>0.178088</td>\n",
       "      <td>4.386685</td>\n",
       "      <td>1.150912e-05</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>O15_MATERN_HYPERT_UNS___O16</th>\n",
       "      <td>PTB</td>\n",
       "      <td>FG: Unspecified maternal hypertension &amp; \\nUK: ...</td>\n",
       "      <td>0.089876</td>\n",
       "      <td>0.615927</td>\n",
       "      <td>0.145920</td>\n",
       "      <td>8.839846e-01</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>GD</td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td>0.410649</td>\n",
       "      <td>0.114888</td>\n",
       "      <td>3.574342</td>\n",
       "      <td>3.511098e-04</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>HP</td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td>0.358103</td>\n",
       "      <td>0.078627</td>\n",
       "      <td>4.554453</td>\n",
       "      <td>5.252199e-06</td>\n",
       "      <td>2</td>\n",
       "      <td>*</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>R18_PAIN_THROAT_CHEST___R07</th>\n",
       "      <td>PTB</td>\n",
       "      <td>FG: Pain in throat and chest &amp; \\nUK: Pain in t...</td>\n",
       "      <td>0.321487</td>\n",
       "      <td>0.220537</td>\n",
       "      <td>1.457746</td>\n",
       "      <td>1.449105e-01</td>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              p1  \\\n",
       "E4_DIABETES___E14             GD   \n",
       "E4_DIABETES___E14             HP   \n",
       "E4_DIABETES___E14            PTB   \n",
       "E4_OBESITY___E66              GD   \n",
       "E4_OBESITY___E66              HP   \n",
       "E4_OBESITY___E66             PTB   \n",
       "I9_HYPTENSESS___I10           GD   \n",
       "I9_HYPTENSESS___I10           HP   \n",
       "I9_HYPTENSESS___I10          PTB   \n",
       "I9_IHD___I25                  GD   \n",
       "I9_IHD___I25                  HP   \n",
       "I9_IHD___I25                 PTB   \n",
       "O15_MATERN_HYPERT_UNS___O16   GD   \n",
       "O15_MATERN_HYPERT_UNS___O16   HP   \n",
       "O15_MATERN_HYPERT_UNS___O16  PTB   \n",
       "R18_PAIN_THROAT_CHEST___R07   GD   \n",
       "R18_PAIN_THROAT_CHEST___R07   HP   \n",
       "R18_PAIN_THROAT_CHEST___R07  PTB   \n",
       "\n",
       "                                                                            p2  \\\n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_DIABETES___E14            FG: Diabetes mellitus & \\nUK: Unspecified diab...   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "E4_OBESITY___E66                                   FG: Obesity & \\nUK: Obesity   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_HYPTENSESS___I10          FG: Hypertension, essential & \\nUK: Essential ...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "I9_IHD___I25                 FG: Ischaemic heart disease, wide definition &...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "O15_MATERN_HYPERT_UNS___O16  FG: Unspecified maternal hypertension & \\nUK: ...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "R18_PAIN_THROAT_CHEST___R07  FG: Pain in throat and chest & \\nUK: Pain in t...   \n",
       "\n",
       "                                   rg        se         z             p  \\\n",
       "E4_DIABETES___E14            0.930488  0.164872  5.643699  1.664346e-08   \n",
       "E4_DIABETES___E14            0.420760  0.098911  4.253925  2.100555e-05   \n",
       "E4_DIABETES___E14            0.300049  0.316877  0.946894  3.436927e-01   \n",
       "E4_OBESITY___E66             0.529482  0.153193  3.456307  5.476317e-04   \n",
       "E4_OBESITY___E66             0.374283  0.092909  4.028490  5.613619e-05   \n",
       "E4_OBESITY___E66             0.175672  0.260471  0.674440  5.000318e-01   \n",
       "I9_HYPTENSESS___I10          0.500904  0.141347  3.543789  3.944202e-04   \n",
       "I9_HYPTENSESS___I10          0.699877  0.101525  6.893642  5.438180e-12   \n",
       "I9_HYPTENSESS___I10          0.319135  0.311526  1.024425  3.056346e-01   \n",
       "I9_IHD___I25                 0.247124  0.091702  2.694859  7.041833e-03   \n",
       "I9_IHD___I25                 0.450153  0.083383  5.398618  6.715603e-08   \n",
       "I9_IHD___I25                 0.148780  0.158187  0.940532  3.469445e-01   \n",
       "O15_MATERN_HYPERT_UNS___O16  0.107861  0.474805  0.227169  8.202923e-01   \n",
       "O15_MATERN_HYPERT_UNS___O16  0.781216  0.178088  4.386685  1.150912e-05   \n",
       "O15_MATERN_HYPERT_UNS___O16  0.089876  0.615927  0.145920  8.839846e-01   \n",
       "R18_PAIN_THROAT_CHEST___R07  0.410649  0.114888  3.574342  3.511098e-04   \n",
       "R18_PAIN_THROAT_CHEST___R07  0.358103  0.078627  4.554453  5.252199e-06   \n",
       "R18_PAIN_THROAT_CHEST___R07  0.321487  0.220537  1.457746  1.449105e-01   \n",
       "\n",
       "                            priority significant  \n",
       "E4_DIABETES___E14                  2           *  \n",
       "E4_DIABETES___E14                  2           *  \n",
       "E4_DIABETES___E14                  2              \n",
       "E4_OBESITY___E66                   2              \n",
       "E4_OBESITY___E66                   2           *  \n",
       "E4_OBESITY___E66                   2              \n",
       "I9_HYPTENSESS___I10                2              \n",
       "I9_HYPTENSESS___I10                2           *  \n",
       "I9_HYPTENSESS___I10                2              \n",
       "I9_IHD___I25                       2              \n",
       "I9_IHD___I25                       2           *  \n",
       "I9_IHD___I25                       2              \n",
       "O15_MATERN_HYPERT_UNS___O16        2              \n",
       "O15_MATERN_HYPERT_UNS___O16        2           *  \n",
       "O15_MATERN_HYPERT_UNS___O16        2              \n",
       "R18_PAIN_THROAT_CHEST___R07        2              \n",
       "R18_PAIN_THROAT_CHEST___R07        2           *  \n",
       "R18_PAIN_THROAT_CHEST___R07        2              "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_to_save = final.loc[:,['p1', 'p2', 'rg', 'se', 'z', 'p', 'priority', 'significant']]\n",
    "data_to_save = data_to_save[data_to_save.priority.astype(str).isin(PRIOR_LIST)]\n",
    "data_to_save.to_csv('./out_data/meta_feature.csv', index=False)\n",
    "data_to_save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fdf7362",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e229d713",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}