raw_data_creation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up and activate virtual environoment and notebook kernel, and install packages necessary for the project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Create a virtual environment\n",
    "# python3 -m venv venv_paper_visualizer\n",
    "\n",
    "# # Activate the virtual environment\n",
    "# source venv_paper_visualizer/bin/activate\n",
    "\n",
    "# # Install packages from requirements.txt\n",
    "# pip install -r requirements.txt\n",
    "\n",
    "# # Add the virtual environment as a Jupyter kernel\n",
    "# python -m ipykernel install --user --name=venv_paper_visualizer --display-name \"Python (venv_paper_visualizer)\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using SemanticScholar API endpoint, retrieve all papers from a year range about a certain topic\n",
    "### The data will be stored in a .jsonl file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Will retrieve an estimated 14882 documents\n",
      "Retrieved 1000 papers...\n",
      "Retrieved 2000 papers...\n",
      "Retrieved 3000 papers...\n",
      "Retrieved 4000 papers...\n",
      "Retrieved 5000 papers...\n",
      "Retrieved 6000 papers...\n",
      "Retrieved 7000 papers...\n",
      "Retrieved 8000 papers...\n",
      "Retrieved 9000 papers...\n",
      "Retrieved 10000 papers...\n",
      "Retrieved 11000 papers...\n",
      "Retrieved 12000 papers...\n",
      "Retrieved 13000 papers...\n",
      "Retrieved 14000 papers...\n",
      "Retrieved 14882 papers...\n",
      "Done! Retrieved 14882 papers total\n",
      "Created papers.jsonl with LLM paper data\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import json\n",
    "\n",
    "query = \"large language model\" \n",
    "# query = \"computational neuroscience\"\n",
    "# first run = 13277\n",
    "# second run = 14144\n",
    "# 12.26.2023 = 14433\n",
    "# 12.29.2023 = 14624 \n",
    "# 12.30.2023 = 14654\n",
    "# 01.01.2024 = 14704\n",
    "# 01.02.2024 = 14773 ??why are 2023 papers still being added??\n",
    "# 01.03.2023 1:45PM = 14793\n",
    "# 01.11.2024 1:01AM = 14862\n",
    "# 01.12.2024 5:29PM = 14882\n",
    "fields = \"paperId,publicationDate,isOpenAccess,openAccessPdf,title,referenceCount,citationCount,influentialCitationCount,abstract,authors\"\n",
    "#field_of_study = \"Computer Science\" # you can filter the category of paper to retrieve\n",
    "min_citation_count=0\n",
    "years = \"2023-2023\"\n",
    "url = f\"http://api.semanticscholar.org/graph/v1/paper/search/bulk?query={query}&fields={fields}&year={years}&minCitationCount={min_citation_count}\"\n",
    "retrieved = 0\n",
    "\n",
    "try:\n",
    "    r = requests.get(url).json()\n",
    "    # Check if 'total' key exists in the response\n",
    "    if 'total' in r:\n",
    "        print(f\"Will retrieve an estimated {r['total']} documents\")\n",
    "    else:\n",
    "        print(\"The 'total' key is not in the response.\")\n",
    "except requests.RequestException as e:\n",
    "    print(f\"An error occurred while making the request: {e}\")\n",
    "except KeyError as e:\n",
    "    print(f\"A key error occurred: {e}\")\n",
    "except Exception as e:\n",
    "    print(f\"An unexpected error occurred: {e}\")\n",
    "\n",
    "with open(f\"papers.jsonl\", \"w\") as file:\n",
    "    while True:\n",
    "        if \"data\" in r:\n",
    "            retrieved += len(r[\"data\"])\n",
    "            print(f\"Retrieved {retrieved} papers...\")\n",
    "            for paper in r[\"data\"]:\n",
    "                print(json.dumps(paper), file=file)\n",
    "        if \"token\" not in r:\n",
    "            break\n",
    "        r = requests.get(f\"{url}&token={r['token']}\").json()\n",
    "\n",
    "print(f\"Done! Retrieved {retrieved} papers total\")\n",
    "print(f\"Created papers.jsonl with LLM paper data\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### You can check how many papers from the semantic shcolar query are actually unique - they all should be..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique paper IDs: 14882\n"
     ]
    }
   ],
   "source": [
    "unique_paper_ids = set()\n",
    "\n",
    "with open('papers.jsonl', 'r') as file:\n",
    "    for line in file:\n",
    "        paper = json.loads(line)\n",
    "        paper_id = paper.get('paperId')\n",
    "        if paper_id:\n",
    "            unique_paper_ids.add(paper_id)\n",
    "\n",
    "print(f\"Number of unique paper IDs: {len(unique_paper_ids)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We want incldue Semantic Scholar's AI generated tldr (\"too long didn't read\" summary) attribute of each paper. Since the bulk papers endpoint used previously doesn't allow us to retreive it, we must use a differnt semantic scholar api endpoint that does."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Added 1000 TLDRs so far.\n",
      "Added 2000 TLDRs so far.\n",
      "Added 3000 TLDRs so far.\n",
      "Added 4000 TLDRs so far.\n",
      "Added 5000 TLDRs so far.\n",
      "Added 6000 TLDRs so far.\n",
      "Added 7000 TLDRs so far.\n",
      "Added 8000 TLDRs so far.\n",
      "Added 9000 TLDRs so far.\n",
      "Added 10000 TLDRs so far.\n",
      "Added 11000 TLDRs so far.\n",
      "Added 12000 TLDRs so far.\n",
      "Added 13000 TLDRs so far.\n",
      "Added 14000 TLDRs so far.\n",
      "Done! Retrieved 14882 TLDRs total.\n",
      "Updated papers with TLDRs in papers.jsonl\n"
     ]
    }
   ],
   "source": [
    "# Read all papers into memory\n",
    "papers = []\n",
    "paper_ids = []\n",
    "with open('papers.jsonl', 'r') as file:\n",
    "    for line in file:\n",
    "        paper = json.loads(line)\n",
    "        papers.append(paper)\n",
    "        paper_id = paper.get('paperId')\n",
    "        if paper_id:\n",
    "            paper_ids.append(paper_id)\n",
    "\n",
    "tldr_count = 0\n",
    "\n",
    "# Fetch TLDRs in batches of 500, the maximum allowed for this endpoint\n",
    "for i in range(0, len(paper_ids), 500):\n",
    "    batch_ids = paper_ids[i:i+500]\n",
    "    response = requests.post(\"https://api.semanticscholar.org/graph/v1/paper/batch\", \n",
    "                             params={'fields': 'tldr'}, \n",
    "                             json={\"ids\": batch_ids})\n",
    "    \n",
    "    if response.status_code == 200:\n",
    "        data = response.json()\n",
    "        tldrs = {paper['paperId']: paper.get('tldr', 'No TLDR found') for paper in data}\n",
    "    else:\n",
    "        print(f\"Failed to fetch TLDRs: {response.status_code}\")\n",
    "        tldrs = {paper_id: 'No TLDR found' for paper_id in batch_ids}\n",
    "\n",
    "    for j in range(i, min(i + 500, len(papers))):\n",
    "        papers[j]['tldr'] = tldrs.get(papers[j]['paperId'], 'No TLDR found')\n",
    "        tldr_count += 1\n",
    "\n",
    "        # Print update every 1000 TLDRs\n",
    "        if tldr_count % 1000 == 0:\n",
    "            print(f\"Added {tldr_count} TLDRs so far.\")\n",
    "\n",
    "with open('papers.jsonl', 'w') as file:\n",
    "    for paper in papers:\n",
    "        file.write(json.dumps(paper) + '\\n')\n",
    "\n",
    "print(f\"Done! Retrieved {tldr_count} TLDRs total.\")\n",
    "print(f\"Updated papers with TLDRs in papers.jsonl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert the .jsonl to a .csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "converted papers.jsonl into papers.csv\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "with open('papers.jsonl', 'r') as jsonl_file, open('papers.csv', 'w', newline='', encoding='utf-8') as csv_file:\n",
    "    csv_writer = csv.writer(csv_file)\n",
    "    # Write the header in the specified order\n",
    "    csv_writer.writerow(['paperId', 'title', 'publicationDate', 'openAccessPdf', 'referenceCount', 'citationCount', 'influentialCitationCount', 'authorNames', 'authorIds', 'abstract', 'tldr'])\n",
    "\n",
    "    for line in jsonl_file:\n",
    "        paper = json.loads(line)\n",
    "\n",
    "        # extract just the url portion\n",
    "        open_access_pdf = paper.get('openAccessPdf', {})\n",
    "        open_access_pdf_url = open_access_pdf.get('url', 'open access PDF is not available') if isinstance(open_access_pdf, dict) else 'open access PDF is not available'\n",
    "\n",
    "        # Extract author names and IDs\n",
    "        author_names = ', '.join([author['name'] for author in paper.get('authors', []) if author.get('name')])\n",
    "        author_ids = ', '.join([str(author['authorId']) for author in paper.get('authors', []) if author.get('authorId')])\n",
    "\n",
    "        # Handle TLDR text\n",
    "        tldr = paper.get('tldr')\n",
    "        tldr_text = tldr.get('text', '') if isinstance(tldr, dict) else ''\n",
    "\n",
    "        # Write the row in the specified order\n",
    "        csv_writer.writerow([\n",
    "            paper.get('paperId', ''),\n",
    "            paper.get('title', ''),\n",
    "            paper.get('publicationDate', ''),\n",
    "            open_access_pdf_url,\n",
    "            paper.get('referenceCount', ''),\n",
    "            paper.get('citationCount', ''),\n",
    "            paper.get('influentialCitationCount', ''),\n",
    "            author_names,\n",
    "            author_ids,\n",
    "            paper.get('abstract', ''),\n",
    "            tldr_text\n",
    "        ])\n",
    "\n",
    "print(\"converted papers.jsonl into papers.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check if the data is messed up and decide how to clean it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "df = pd.read_csv('papers.csv')\n",
    "\n",
    "missing_abstract_and_tldr = df[pd.isna(df['abstract']) & pd.isna(df['tldr'])][['paperId', 'publicationDate', 'influentialCitationCount', 'title', 'tldr', 'abstract']].set_index('paperId').T.to_dict()\n",
    "\n",
    "abstract_na_tldr_questionable = df[pd.isna(df['abstract']) & (df['tldr'].str.len() < 150)][['paperId', 'publicationDate', 'influentialCitationCount', 'title', 'tldr', 'abstract']].set_index('paperId').T.to_dict()\n",
    "\n",
    "abstract_questionable = df[(df['abstract'].str.len() < 300)][['paperId', 'publicationDate', 'influentialCitationCount', 'title', 'tldr', 'abstract']].set_index('paperId').T.to_dict()\n",
    "\n",
    "tldr_questionable = df[(df['tldr'].str.len() < 100)][['paperId', 'publicationDate', 'influentialCitationCount', 'title', 'tldr', 'abstract']].set_index('paperId').T.to_dict()\n",
    "\n",
    "# Create the directory if it does not exist\n",
    "directory = 'questionable_data'\n",
    "if not os.path.exists(directory):\n",
    "    os.makedirs(directory)\n",
    "\n",
    "with open('questionable_data/missing_abstract_and_tldr.txt', 'w') as file:\n",
    "    for paper_id, info in missing_abstract_and_tldr.items():\n",
    "        file.write(f\"paperId: {paper_id}\\n\")\n",
    "        file.write(f\"publicationDate: {info['publicationDate']}\\n\")\n",
    "        file.write(f\"influentialCitationCount: {info['influentialCitationCount']}\\n\")\n",
    "        file.write(f\"title: {info['title']}\\n\")\n",
    "        file.write(f\"tldr: {info['tldr']}\\n\")\n",
    "        file.write(f\"abstract: {info['abstract']}\\n\\n\")\n",
    "\n",
    "with open('questionable_data/abstract_na_tldr_questionable.txt', 'w') as file:\n",
    "    for paper_id, info in abstract_na_tldr_questionable.items():\n",
    "        file.write(f\"paperId: {paper_id}\\n\")\n",
    "        file.write(f\"publicationDate: {info['publicationDate']}\\n\")\n",
    "        file.write(f\"influentialCitationCount: {info['influentialCitationCount']}\\n\")\n",
    "        file.write(f\"title: {info['title']}\\n\")\n",
    "        file.write(f\"tldr: {info['tldr']}\\n\")\n",
    "        file.write(f\"abstract: {info['abstract']}\\n\\n\")\n",
    "\n",
    "with open('questionable_data/abstract_questionable.txt', 'w') as file:\n",
    "    for paper_id, info in abstract_questionable.items():\n",
    "        file.write(f\"paperId: {paper_id}\\n\")\n",
    "        file.write(f\"publicationDate: {info['publicationDate']}\\n\")\n",
    "        file.write(f\"influentialCitationCount: {info['influentialCitationCount']}\\n\")\n",
    "        file.write(f\"title: {info['title']}\\n\")\n",
    "        file.write(f\"tldr: {info['tldr']}\\n\")\n",
    "        file.write(f\"abstract: {info['abstract']}\\n\\n\")\n",
    "\n",
    "with open('questionable_data/tldr_questionable.txt', 'w') as file:\n",
    "    for paper_id, info in tldr_questionable.items():\n",
    "        file.write(f\"paperId: {paper_id}\\n\")\n",
    "        file.write(f\"publicationDate: {info['publicationDate']}\\n\")\n",
    "        file.write(f\"influentialCitationCount: {info['influentialCitationCount']}\\n\")\n",
    "        file.write(f\"title: {info['title']}\\n\")\n",
    "        file.write(f\"tldr: {info['tldr']}\\n\")\n",
    "        file.write(f\"abstract: {info['abstract']}\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(abstract_questionable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do final data preprocessing. The goal is to make sure every paper has a sufficiently long abstract that will be the basis for clustering. It's also important for the tldr to be present because that is what will be eventually displayed in RShiny plotly visualization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Papers to start with: 14882\n",
      "\n",
      "Number of papers where both tldr and abstract are NA (dropped): 487\n",
      "Number of rows with questionable abstracts imputed with title + tldr + abstract: 78\n",
      "Number of rows with missing abstracts imputed with title + tldr: 530\n",
      "Number of rows with tldr imputed: 827\n",
      "\n",
      "Papers left: 14395\n"
     ]
    }
   ],
   "source": [
    "# 1. Drop rows where both tldr and abstract is NA value\n",
    "df = pd.read_csv('papers.csv')\n",
    "initial_row_count = df.shape[0]\n",
    "print(f\"Papers to start with: {initial_row_count}\\n\")\n",
    "df = df.dropna(subset=['abstract', 'tldr'], how='all')\n",
    "final_row_count = df.shape[0]\n",
    "rows_dropped = initial_row_count - final_row_count\n",
    "print(f\"Number of papers where both tldr and abstract are NA (dropped): {rows_dropped}\")\n",
    "df.to_csv('papers.csv', index=False)\n",
    "\n",
    "# 2. If abstract is NA and tldr is fucked, drop that row --> judge this from \"abstract_na_tldr_questionable.txt\"\n",
    "# ... the observations in this file look good\n",
    "\n",
    "# 3. If abstract is questionable (abnormally short), impute abstract with the title concatenated with tldr and with abstract\n",
    "# Note that some of these observations should be straight up dropped, be we not gonna spend time manually inpecting that shit\n",
    "df = pd.read_csv('papers.csv')\n",
    "imputed_rows = 0\n",
    "for index, row in df.iterrows():\n",
    "    if not pd.isna(row['abstract']) and len(row['abstract']) < 300:\n",
    "        tldr = '' if pd.isna(row['tldr']) else row['tldr']\n",
    "        df.at[index, 'abstract'] = row['title'] + \". \" + tldr + \". \" + row['abstract']\n",
    "        imputed_rows += 1\n",
    "print(f\"Number of rows with questionable abstracts imputed with title + tldr + abstract: {imputed_rows}\")\n",
    "df.to_csv('papers.csv', index=False)\n",
    "\n",
    "# 4. If abstract is NA and tldr is ok, impute abstract with the tldr concatenated with title\n",
    "# !! note that this step has to come after the last one, other wise abstract will be double concatenated\n",
    "df = pd.read_csv('papers.csv')\n",
    "na_abstract_count = df['abstract'].isna().sum()\n",
    "df.loc[df['abstract'].isna(), 'abstract'] = df['title'] + \". \" + df['tldr']\n",
    "print(f\"Number of rows with missing abstracts imputed with title + tldr: {na_abstract_count}\")\n",
    "df.to_csv('papers.csv', index=False)\n",
    "\n",
    "# 5. If abstract is ok (after steps 1-4 it should be) and tldr is NA, impute tldr with abstract\n",
    "df = pd.read_csv('papers.csv')\n",
    "na_tldr_count = df['tldr'].isna().sum()\n",
    "df.loc[df['tldr'].isna(), 'tldr'] = df['abstract']\n",
    "print(f\"Number of rows with tldr imputed: {na_tldr_count}\")\n",
    "df.to_csv('papers.csv', index=False)\n",
    "\n",
    "# 6. Drop rows where publicationDate is NA - it doesn't make sense to keep these rows since the\n",
    "# RShiny app has a plot of paper topic trends over time. Also, these extra ~1500 observations slow down\n",
    "# RShiny app performance. However, this leaves many good papers on the table, so use discretion.\n",
    "# df = pd.read_csv('papers.csv')\n",
    "# initial_row_count = df.shape[0]\n",
    "# df = df.dropna(subset=['publicationDate'])\n",
    "# final_row_count = df.shape[0]\n",
    "# rows_dropped = initial_row_count - final_row_count\n",
    "# print(f\"Number of rows dropped where publicationDate was NA (dropped): {rows_dropped}\")\n",
    "# df.to_csv('papers.csv', index=False)\n",
    "\n",
    "# Print final paper count\n",
    "print(f\"\\nPapers left: {df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now it's time for BertTOPIC. The first step is to create vector embeddings of the abstract of each article and provide it as an \"embedding_model\" to the BertTOPIC processes that will follow. This model contains both the vector representations of the abstracts along with the text of the abstract.\n",
    "### I found that using the abstract instead of the tldr produces more accurate clusters, a result that is somewhat surprising given the extra noise present in abstracts compared to the concise tldrs.\n",
    "### The code block below is CPU intensive - ONLY RUN ONCE for a particular Semantic Scholar search query. Once the .json embeddings file is created, you can run the block below the next mardown cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 405/405 [27:53<00:00,  4.13s/it] \n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('papers.csv')\n",
    "abstracts = df['abstract'].tolist()\n",
    "paper_ids = df['paperId'].tolist()\n",
    "\n",
    "embedding_model = SentenceTransformer(\"BAAI/bge-small-en\")\n",
    "abstract_embeddings = embedding_model.encode(abstracts, show_progress_bar=True)\n",
    "\n",
    "##### It takes 30 minutes to generate abstract embeddings for 13k abstracts on my 16 GB macbook pro.\n",
    "##### It is useful to save these to a file in case you need them later.\n",
    "\n",
    "# map paperId to its abstract embedding\n",
    "paperId_to_embedding = dict(zip(paper_ids, abstract_embeddings))\n",
    "# Convert numpy arrays in the dictionary to lists for JSON serialization\n",
    "for paper_id in paperId_to_embedding:\n",
    "    paperId_to_embedding[paper_id] = paperId_to_embedding[paper_id].tolist()\n",
    "# Save the dictionary as a JSON file\n",
    "with open('abstract_embeddings.json', 'w') as file:\n",
    "    json.dump(paperId_to_embedding, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run these jawns to quickly update abstract_embeddings from an exisiting embeddings file in case your papers.csv file had a paper update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are differences between the paperId sets.\n",
      "IDs in CSV but not in JSON: 166\n",
      "IDs in JSON but not in CSV: 53\n",
      "Sample IDs in CSV but not in JSON: ['3f2c6aa0b347f10c02ee6b5e81d857f003bf3e43', '446c951de45439c3e53903916bf1b85c0475109e', 'ac47bd3b512301371fc87c68416befce6589912e', '80e642da57d4c7d4d36a770a810268e37556de27', '071404ea62a7e32d93075a570579e438596a970b', '5650b7243d27a91fe10c68bc84c80197cddaa8dc', 'a6868c28bfb842930e31216441c05526a2c6f752', 'bcaaf83946ffe764e3405799fae739582082bc6f', 'b411c8f98865565f54642af1a5c010bde6beaedc', '13eacc692aeb58c7987c535c439eeb345076bea2']\n",
      "Sample IDs in JSON but not in CSV: ['59b9f84dccf23c8262b09db62e0b0a57c864b02b', '71d206e927017ff4a5de71b7bd1b0fa02a23ad9f', '6483a6f2038cd8583ad5b6678602bc904459a7f7', '9ff92d31babb7bdaecf7220b0a81c701230d8b95', 'd5c2947cab82c44e3cca8e90486da10a81e1f697', 'cc7658bd47cac361abad0b42b71a69b35ab6a23a', '0a29c1be5a2ecdf57f4615d7fe0201b176706439', '655eeac08f2c81f4e62ce72322a31af9ee11a9c2', '65e1efa56dea7067d0631304e81633c719b8ba74', 'baa1b6de7c94e3ce238116f8b3c31603c6bb8157']\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "df = pd.read_csv('papers.csv')\n",
    "\n",
    "# Ensure paperId is treated as a string and strip any whitespace\n",
    "df['paperId'] = df['paperId'].astype(str).str.strip()\n",
    "paper_ids_csv = set(df['paperId'])\n",
    "\n",
    "# Load the JSON file\n",
    "with open('abstract_embeddings.json', 'r') as file:\n",
    "    existing_embeddings = json.load(file)\n",
    "\n",
    "# Ensure keys are treated as strings\n",
    "paper_ids_json = set(map(str, existing_embeddings.keys()))\n",
    "\n",
    "# Check if the sets are exactly the same\n",
    "if paper_ids_csv == paper_ids_json:\n",
    "    print(\"The paperId sets from CSV and JSON are identical.\")\n",
    "else:\n",
    "    print(\"There are differences between the paperId sets.\")\n",
    "\n",
    "    # Find and display differences\n",
    "    diff_csv_not_json = paper_ids_csv - paper_ids_json\n",
    "    diff_json_not_csv = paper_ids_json - paper_ids_csv\n",
    "\n",
    "    print(f\"IDs in CSV but not in JSON: {len(diff_csv_not_json)}\")\n",
    "    print(f\"IDs in JSON but not in CSV: {len(diff_json_not_csv)}\")\n",
    "\n",
    "    # Print some sample differences for inspection\n",
    "    print(\"Sample IDs in CSV but not in JSON:\", list(diff_csv_not_json)[:10])\n",
    "    print(\"Sample IDs in JSON but not in CSV:\", list(diff_json_not_csv)[:10])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davydsadovskyy/semantic-scholar-visualizer/venv_paper_visualizer/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of new papers: 166\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 6/6 [00:25<00:00,  4.22s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Appended 166 new embeddings.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import numpy as np\n",
    "\n",
    "### Step 1: Load existing embeddings\n",
    "with open('abstract_embeddings.json', 'r') as file:\n",
    "    existing_embeddings = json.load(file)\n",
    "# Convert them back to numpy arrays\n",
    "for paper_id in existing_embeddings:\n",
    "    existing_embeddings[paper_id] = np.array(existing_embeddings[paper_id])\n",
    "\n",
    "### Step 2: Load the DataFrame and identify new papers\n",
    "df = pd.read_csv('papers.csv')\n",
    "paper_ids = df['paperId'].tolist()\n",
    "abstracts = df['abstract'].tolist()\n",
    "new_paper_ids = [pid for pid in paper_ids if pid not in existing_embeddings]\n",
    "print(f\"Number of new papers: {len(new_paper_ids)}\")\n",
    "\n",
    "# Step 3: Generate embeddings for new papers if there are any\n",
    "if new_paper_ids:\n",
    "    new_abstracts = df[df['paperId'].isin(new_paper_ids)]['abstract'].tolist()\n",
    "    \n",
    "    embedding_model = SentenceTransformer(\"BAAI/bge-small-en\")\n",
    "    new_embeddings = embedding_model.encode(new_abstracts, show_progress_bar=True)\n",
    "\n",
    "    # Update existing_embeddings with new embeddings\n",
    "    for pid, emb in zip(new_paper_ids, new_embeddings):\n",
    "        existing_embeddings[pid] = emb\n",
    "\n",
    "    # Save the updated embeddings\n",
    "    with open('abstract_embeddings.json', 'w') as file:\n",
    "        json.dump({pid: emb.tolist() for pid, emb in existing_embeddings.items()}, file)\n",
    "    \n",
    "    print(f\"Appended {len(new_paper_ids)} new embeddings.\")\n",
    "\n",
    "# Step 4: Create a numpy array of embeddings in the order of paperIds in papers.csv\n",
    "abstract_embeddings = np.array([existing_embeddings[pid] for pid in paper_ids], dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the clustering and dimensionality reduction models used in bertTOPIC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from umap import UMAP\n",
    "from hdbscan import HDBSCAN\n",
    "\n",
    "# technique that helps in reducing the high-dimensional embeddings (generated from the text data)\n",
    "# into a lower-dimensional space. This step is crucial because it makes the clustering step that\n",
    "# follows more manageable and can improve the quality of the clusters found by the clustering algorithm.\n",
    "umap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=12345)\n",
    "\n",
    "# clustering algorithm that identifies clusters of points in the reduced dimensionality space.\n",
    "# It works well with how UMAP structures the data and is capable of finding clusters of varying densities.\n",
    "# I've found that a cluster size of .003 the size of the number of data points works best for creating more\n",
    "# evenly distributed categories.\n",
    "hdbscan_model = HDBSCAN(min_cluster_size=int(len(abstracts) * .003), metric='euclidean', cluster_selection_method='eom', prediction_data=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the representation models used for bertTOPIC. These are what actually make sense of the clusters and label them with words or phrases. There are many options for these. You can even use the OpenAI bertTOPIC integration to create a summary of your liking. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bertopic.representation import KeyBERTInspired, MaximalMarginalRelevance, OpenAI\n",
    "import openai\n",
    "import tiktoken\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "openai_key = os.getenv('OPENAI_KEY')\n",
    "\n",
    "# models that make \"hard to read\" labels for created topics\n",
    "keybert = KeyBERTInspired()\n",
    "mmr = MaximalMarginalRelevance(diversity=0.3)\n",
    "\n",
    "# openAI model that makes \"easy to read\" labels for created topics\n",
    "client = openai.OpenAI(api_key=openai_key)\n",
    "tokenizer= tiktoken.encoding_for_model(\"gpt-3.5-turbo\")\n",
    "label_prompt = \"\"\"\n",
    "# CONTEXT #\n",
    "I performed clustering analysis on 10000 academic papers about large language models. The clustering \n",
    "algorithm found 50 clusters in which to group these papers. The algorithm returned information about the\n",
    "clusters as a set of keywords that represent the cluster, along with a set of 4 abstracts that are at the\n",
    "\"center\" of the cluster\n",
    "\n",
    "# OBJECTIVE #\n",
    "I need to generate a short label that will be fed into a dashboard to represent the cluster. The label needs to \n",
    "capture the essense of the cluster. All I have to understand what the cluster is about is these keywords and\n",
    "abstracts that the clustering algorithm returned: ////[KEYWORDS] [DOCUMENTS]////\n",
    "\n",
    "# OUTPUT # \n",
    "Just the short topic label and no other output whatsoever. Use abberviations on for things like \"large language\n",
    "models\" or \"artificial intelligence\", but don't abreviate other terms. The first word in the label cannot be \"LLM\".\n",
    "\"\"\"\n",
    "gpt_label = OpenAI(\n",
    "    client,\n",
    "    prompt=label_prompt,\n",
    "    diversity=0.3,\n",
    "    model=\"gpt-4-1106-preview\", \n",
    "    delay_in_seconds=2, \n",
    "    chat=True,\n",
    "    nr_docs=4,\n",
    "    doc_length=800,\n",
    "    tokenizer=tokenizer\n",
    ")\n",
    "\n",
    "# Optional - create a summary for each cluster\n",
    "# summary_prompt = \"\"\"\n",
    "# I have a topic that is described by the following keywords: [KEYWORDS]\n",
    "# The following are some documents that most closely represent the topic: \n",
    "# [DOCUMENTS]\n",
    "\n",
    "# Based on the information above, capture the essense of these 4 documents as a short summary no longer than 80 words.\n",
    "# You must adhere to this word limit. Also, you must provide output in the following format: topic: <topic summary>\n",
    "# You MUST make sure not to overly rely on information from any one document.\n",
    "# \"\"\"\n",
    "# gpt_summary = OpenAI(\n",
    "#     client,\n",
    "#     prompt=summary_prompt,\n",
    "#     diversity=0.3,\n",
    "#     model=\"gpt-3.5-turbo\", \n",
    "#     delay_in_seconds=2, \n",
    "#     chat=True,\n",
    "#     nr_docs=4,\n",
    "#     doc_length=800,\n",
    "#     tokenizer=tokenizer\n",
    "# )\n",
    "\n",
    "\n",
    "# I will use 3 difference representation models\n",
    "representation_model = {\n",
    "    \"GPTLabel\": gpt_label,\n",
    "    #\"GPTSummary\": gpt_summary,\n",
    "    \"KeyBERT\": keybert,\n",
    "    \"MMR\": mmr\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Perform the clustering and labeling with BERTopic using all the models we just defined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-12 17:35:30,412 - BERTopic - Dimensionality - Fitting the dimensionality reduction algorithm\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-12 17:36:01,396 - BERTopic - Dimensionality - Completed ✓\n",
      "2024-01-12 17:36:01,397 - BERTopic - Cluster - Start clustering the reduced embeddings\n",
      "2024-01-12 17:36:02,145 - BERTopic - Cluster - Completed ✓\n",
      "2024-01-12 17:36:02,149 - BERTopic - Representation - Extracting topics from clusters using representation models.\n",
      "  0%|          | 0/46 [00:00<?, ?it/s]huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "100%|██████████| 46/46 [02:33<00:00,  3.34s/it]\n",
      "2024-01-12 17:39:21,153 - BERTopic - Representation - Completed ✓\n"
     ]
    }
   ],
   "source": [
    "from bertopic import BERTopic\n",
    "\n",
    "# Initialize BERTopic with the sub-models\n",
    "topic_model = BERTopic(\n",
    "                        # Sub-models\n",
    "                        embedding_model=embedding_model,\n",
    "                        umap_model=umap_model,\n",
    "                        hdbscan_model=hdbscan_model,\n",
    "                        representation_model=representation_model,\n",
    "                        # Hyperparameters\n",
    "                        top_n_words=10,\n",
    "                        verbose=True)\n",
    "\n",
    "# Train the BERTopic model\n",
    "topics, probs = topic_model.fit_transform(abstracts, abstract_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output the GPT generated labels to a .txt file to see if they look acceptable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = topic_model.get_topic_info()\n",
    "gpt_labels = df['GPTLabel']\n",
    "with open('gpt_labels.txt', 'w') as file:\n",
    "    for label in gpt_labels:\n",
    "        file.write(str(label) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic</th>\n",
       "      <th>Count</th>\n",
       "      <th>Name</th>\n",
       "      <th>Representation</th>\n",
       "      <th>GPTLabel</th>\n",
       "      <th>KeyBERT</th>\n",
       "      <th>MMR</th>\n",
       "      <th>Representative_Docs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1</td>\n",
       "      <td>3952</td>\n",
       "      <td>-1_the_of_and_to</td>\n",
       "      <td>[the, of, and, to, in, we, that, for, language...</td>\n",
       "      <td>[NLP Applications &amp; Challenges of LLMs]</td>\n",
       "      <td>[nlp, ai, language, model, models, chatgpt, ll...</td>\n",
       "      <td>[the, of, and, to, in, we, that, for, language...</td>\n",
       "      <td>[This paper presents a comprehensive and pract...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1756</td>\n",
       "      <td>0_and_medical_of_the</td>\n",
       "      <td>[and, medical, of, the, in, clinical, to, were...</td>\n",
       "      <td>[ChatGPT's Role in Medical Education and Clini...</td>\n",
       "      <td>[chatgpt, medicine, research, nlp, medical, in...</td>\n",
       "      <td>[and, medical, of, the, in, clinical, to, were...</td>\n",
       "      <td>[1 N o w a d a y s , t h e c o n c e p t o f a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1480</td>\n",
       "      <td>1_image_visual_the_to</td>\n",
       "      <td>[image, visual, the, to, vision, and, video, w...</td>\n",
       "      <td>[Multimodal LLM Integration &amp; Applications]</td>\n",
       "      <td>[multimodal, model, models, captioning, langua...</td>\n",
       "      <td>[image, visual, the, to, vision, and, video, w...</td>\n",
       "      <td>[The exponential growth of large language mode...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>532</td>\n",
       "      <td>2_students_ai_and_the</td>\n",
       "      <td>[students, ai, and, the, of, chatgpt, educatio...</td>\n",
       "      <td>[AI Chatbots in Education]</td>\n",
       "      <td>[chatgpt, ai, research, researchers, learning,...</td>\n",
       "      <td>[students, ai, and, the, of, chatgpt, educatio...</td>\n",
       "      <td>[The fear of whether artificial intelligence (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>511</td>\n",
       "      <td>3_translation_languages_the_language</td>\n",
       "      <td>[translation, languages, the, language, of, mo...</td>\n",
       "      <td>[Machine Translation Enhancement Techniques]</td>\n",
       "      <td>[translation, nlp, multilingual, language, mon...</td>\n",
       "      <td>[translation, languages, the, language, of, mo...</td>\n",
       "      <td>[. In recent times, our research has focused o...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Topic  Count                                  Name  \\\n",
       "0     -1   3952                      -1_the_of_and_to   \n",
       "1      0   1756                  0_and_medical_of_the   \n",
       "2      1   1480                 1_image_visual_the_to   \n",
       "3      2    532                 2_students_ai_and_the   \n",
       "4      3    511  3_translation_languages_the_language   \n",
       "\n",
       "                                      Representation  \\\n",
       "0  [the, of, and, to, in, we, that, for, language...   \n",
       "1  [and, medical, of, the, in, clinical, to, were...   \n",
       "2  [image, visual, the, to, vision, and, video, w...   \n",
       "3  [students, ai, and, the, of, chatgpt, educatio...   \n",
       "4  [translation, languages, the, language, of, mo...   \n",
       "\n",
       "                                            GPTLabel  \\\n",
       "0            [NLP Applications & Challenges of LLMs]   \n",
       "1  [ChatGPT's Role in Medical Education and Clini...   \n",
       "2        [Multimodal LLM Integration & Applications]   \n",
       "3                         [AI Chatbots in Education]   \n",
       "4       [Machine Translation Enhancement Techniques]   \n",
       "\n",
       "                                             KeyBERT  \\\n",
       "0  [nlp, ai, language, model, models, chatgpt, ll...   \n",
       "1  [chatgpt, medicine, research, nlp, medical, in...   \n",
       "2  [multimodal, model, models, captioning, langua...   \n",
       "3  [chatgpt, ai, research, researchers, learning,...   \n",
       "4  [translation, nlp, multilingual, language, mon...   \n",
       "\n",
       "                                                 MMR  \\\n",
       "0  [the, of, and, to, in, we, that, for, language...   \n",
       "1  [and, medical, of, the, in, clinical, to, were...   \n",
       "2  [image, visual, the, to, vision, and, video, w...   \n",
       "3  [students, ai, and, the, of, chatgpt, educatio...   \n",
       "4  [translation, languages, the, language, of, mo...   \n",
       "\n",
       "                                 Representative_Docs  \n",
       "0  [This paper presents a comprehensive and pract...  \n",
       "1  [1 N o w a d a y s , t h e c o n c e p t o f a...  \n",
       "2  [The exponential growth of large language mode...  \n",
       "3  [The fear of whether artificial intelligence (...  \n",
       "4  [. In recent times, our research has focused o...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_model.get_topic_info()[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add each paper's assigned category into our papers.csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = topic_model.get_topic_info()\n",
    "topic_dict = {row['Topic']: 'Unassigned Papers' if row['Topic'] == -1 else row['GPTLabel'] for index, row in df.iterrows()}\n",
    "for key in topic_dict:\n",
    "    if isinstance(topic_dict[key], list):\n",
    "        topic_dict[key] = topic_dict[key][0] if topic_dict[key] else 'No Label'\n",
    "\n",
    "\n",
    "df = pd.read_csv('papers.csv')\n",
    "df['topicNumber'] = topics\n",
    "df['topicLabel'] = df['topicNumber'].map(topic_dict)\n",
    "\n",
    "df.to_csv('papers.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create reduced vector embeddings of the paper abstracts for RShiny visualization purposes, and save them to our papers.csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "reducedEmbeddings3D = UMAP(n_neighbors=15, n_components=3, min_dist=0.0, metric='cosine', random_state=12345).fit_transform(abstract_embeddings)\n",
    "reducedEmbeddings2D = UMAP(n_neighbors=15, n_components=2, min_dist=0.0, metric='cosine', random_state=12345).fit_transform(abstract_embeddings)\n",
    "\n",
    "df = pd.read_csv('papers.csv')\n",
    "df['x3D'] = reducedEmbeddings3D[:, 0]\n",
    "df['y3D'] = reducedEmbeddings3D[:, 1]\n",
    "df['z3D'] = reducedEmbeddings3D[:, 2]\n",
    "df['x2D'] = reducedEmbeddings2D[:, 0]\n",
    "df['y2D'] = reducedEmbeddings2D[:, 1]\n",
    "df.to_csv('papers.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove columns from papers.csv that will not be used in the RShiny dashboard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('papers.csv')\n",
    "df = df.drop(columns=['abstract', 'topicNumber'])\n",
    "df.to_csv('papers.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv_paper_visualizer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}