ZSFG_Demographics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7313bc91",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import regex as re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "db78cf66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60049"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ZSFG_reports = pd.read_csv('data/raw/secure_ZSFG_radreports__CT_MR_US_NM__01-01-2022__12-29-2022.csv')\n",
    "ZSFG_reports = ZSFG_reports[ZSFG_reports['Modality'].isin(['CT', 'MR', 'US'])].reset_index(drop=True)\n",
    "ZSFG_reports['Report Text'] = ZSFG_reports['Report Text'].str.replace('_x000D_', '').str.replace('\\t', ' ')\n",
    "len(ZSFG_reports)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43d57d05",
   "metadata": {},
   "source": [
    "## Exclude Studies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d18d8078",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n = 2\n"
     ]
    }
   ],
   "source": [
    "# Reports with same accessions\n",
    "ZSFG_duplicates = ZSFG_reports[ZSFG_reports.duplicated(subset='Accession Number', keep=False)]\n",
    "ZSFG_reports = ZSFG_reports.drop_duplicates(subset='Accession Number', keep=False).reset_index(drop=True)\n",
    "print('n =', len(ZSFG_duplicates))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d5065a47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n = 0\n"
     ]
    }
   ],
   "source": [
    "# Reports that are non-reportable studies \n",
    "ZSFG_nonreportable_reports = ZSFG_reports[ZSFG_reports['Report Text'].str.contains('non-reportable study')].reset_index(drop=True)\n",
    "ZSFG_reports = ZSFG_reports[~ZSFG_reports['Report Text'].str.contains('non-reportable study')].reset_index(drop=True)\n",
    "print('n =', len(ZSFG_nonreportable_reports))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "636d213a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n = 124\n"
     ]
    }
   ],
   "source": [
    "# Reports with no separate findings and impressions\n",
    "def extract_impression(report_text):\n",
    "    pattern = r'(Impression:|IMPRESSION:|IMPRESSION:\\n)(.*?)(The above|Wet read|Findings are|Electronically|REFERENCE:)'\n",
    "    match = re.search(pattern, report_text, re.DOTALL)\n",
    "    if match:\n",
    "        return match.group(2).strip()\n",
    "    return ''\n",
    "\n",
    "def extract_source(report_text):\n",
    "    pattern = r'^(.*?)(Impression:|IMPRESSION:|IMPRESSION:\\n)'\n",
    "\n",
    "    match = re.search(pattern, report_text, re.DOTALL)\n",
    "    if match:\n",
    "        return match.group(1).strip()  \n",
    "    return '' \n",
    "\n",
    "ZSFG_reports['Processed Impression'] = ZSFG_reports['Report Text'].apply(extract_impression)\n",
    "ZSFG_reports['Processed Source'] = ZSFG_reports['Report Text'].apply(extract_source)\n",
    "\n",
    "ZSFG_no_separate_impression = ZSFG_reports[\n",
    "    (ZSFG_reports['Processed Impression'] == '') | (ZSFG_reports['Processed Source'] == '')\n",
    "]\n",
    "ZSFG_reports = ZSFG_reports[\n",
    "    (ZSFG_reports['Processed Impression'] != '') & (ZSFG_reports['Processed Source'] != '')\n",
    "].reset_index(drop=True)\n",
    "print('n =', len(ZSFG_no_separate_impression))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d9603eda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n = 59923\n",
      "27530 patients\n"
     ]
    }
   ],
   "source": [
    "print('n =', len(ZSFG_reports))\n",
    "print(len(ZSFG_reports['Patient MRN'].unique()), 'patients')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d820b538",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Age (y)\n",
      "====================\n",
      "52.620025532767045 +/- 19.308107764202852\n"
     ]
    }
   ],
   "source": [
    "print('Age (y)')\n",
    "print('='*20)\n",
    "print(ZSFG_reports['Patient Age'].mean(), '+/-', ZSFG_reports['Patient Age'].std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "225acd3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Patient Sex\n",
      "====================\n",
      "Male      0.536305\n",
      "Female    0.463261\n",
      "U         0.000434\n",
      "Name: Patient Sex, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print('Patient Sex')\n",
    "print('='*20)\n",
    "ZSFG_reports['Patient Sex'] = ZSFG_reports['Patient Sex'].fillna('U')\n",
    "print(ZSFG_reports['Patient Sex'].value_counts(normalize=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "13281706",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Modality\n",
      "====================\n",
      "CT     0.611451\n",
      "US     0.262086\n",
      "MRI    0.126462\n",
      "Name: Modality, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print('Modality')\n",
    "print('='*20)\n",
    "print(ZSFG_reports['Modality'].value_counts(normalize=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "71d8809c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Patient Status\n",
      "====================\n",
      "Outpatient    40.313402\n",
      "Emergency     34.694525\n",
      "Inpatient     24.818517\n",
      "Other          0.173556\n",
      "Name: Patient Status, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print('Patient Status')\n",
    "print('='*20)\n",
    "ZSFG_reports['Patient Status'] = ZSFG_reports['Patient Status'].fillna('Other')\n",
    "print(ZSFG_reports['Patient Status'].value_counts(normalize=True)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "620403ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is Stat\n",
      "====================\n",
      "True     0.600237\n",
      "False    0.399763\n",
      "Name: Is Stat, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print('Is Stat')\n",
    "print('='*20)\n",
    "print(ZSFG_reports['Is Stat'].value_counts(normalize=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "9aa6f37d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Body Part Imaged\n",
      "====================\n",
      "ABDOMEN/PELVIS    0.268511\n",
      "BRAIN             0.204145\n",
      "OTHER             0.196352\n",
      "CHEST             0.153230\n",
      "NECK              0.052017\n",
      "SPINE             0.037081\n",
      "RENAL/KIDNEY      0.022863\n",
      "EXTREMITY         0.020526\n",
      "HEAD              0.018324\n",
      "LIVER             0.017973\n",
      "KNEE              0.006792\n",
      "HIP               0.002069\n",
      "BREAST            0.000117\n",
      "Name: Body Part Imaged, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "def label_body_part(exam_description):\n",
    "    if 'ABDOMEN' in exam_description or 'PELVIS' in exam_description:\n",
    "        return 'ABDOMEN/PELVIS'\n",
    "    elif 'CHEST' in exam_description:\n",
    "        return 'CHEST'\n",
    "    elif 'BRAIN' in exam_description:\n",
    "        return 'BRAIN'\n",
    "    elif 'NECK' in exam_description:\n",
    "        return 'NECK'\n",
    "    elif 'SPINE' in exam_description or 'SPINAL' in exam_description:\n",
    "        return 'SPINE'\n",
    "    elif 'EXTREMITY' in exam_description:\n",
    "        return 'EXTREMITY'\n",
    "    elif 'KIDNEY' in exam_description or 'RENAL' in exam_description or 'Kidneys' in exam_description:\n",
    "        return 'RENAL/KIDNEY'\n",
    "    elif 'BREAST' in exam_description:\n",
    "        return 'BREAST'\n",
    "    elif 'LIVER' in exam_description:\n",
    "        return 'LIVER'\n",
    "    elif 'PROSTATE' in exam_description:\n",
    "        return 'PROSTATE'  \n",
    "    elif 'KNEE' in exam_description:\n",
    "        return 'KNEE'  \n",
    "    elif 'HEAD' in exam_description or 'Head' in exam_description:\n",
    "        return 'HEAD'\n",
    "    elif 'HIP' in exam_description:\n",
    "        return 'HIP'\n",
    "    elif 'HEART' in exam_description:\n",
    "        return 'HEART'\n",
    "    \n",
    "    return 'OTHER'\n",
    "    \n",
    "print('Body Part Imaged')\n",
    "print('='*20)\n",
    "ZSFG_reports['Body Part Imaged'] = ZSFG_reports['Exam Description'].apply(label_body_part)\n",
    "print(ZSFG_reports['Body Part Imaged'].value_counts(normalize=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b97d891b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CT     36640\n",
       "US     15705\n",
       "MRI     7578\n",
       "Name: Modality, dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ZSFG_reports['Modality'] = ZSFG_reports['Modality'].replace('MR', 'MRI')\n",
    "ZSFG_reports['Modality'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3075a152",
   "metadata": {},
   "outputs": [],
   "source": [
    "ZSFG_reports.to_csv(f'Flan_T5_Finetune/data/ZSFG_test.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "report_gen",
   "language": "python",
   "name": "report_gen"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}