arxiv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6809876d",
   "metadata": {},
   "source": [
    "# arXiv paper collection\n",
    "\n",
    "### To run this notebook, you need to install the following packages:\n",
    "\n",
    "- ```arxiv```\n",
    "- ```pandas```\n",
    "\n",
    "### The notebook is sectioned into the following parts:\n",
    "\n",
    "- [Metadata and fulltext collection](#Metadata-and-fulltext-collection)\n",
    "- [Metadata exploration](#Metadata-exploration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "932ae673",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Metadata and fulltext collection\n",
    "\n",
    "import arxiv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f9b400e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Metadata exploration\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "596d04bd",
   "metadata": {},
   "source": [
    "## Metadata and fulltext collection\n",
    "\n",
    "- [Back to top](#arXiv-paper-collection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "624af607",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating Search instance\n",
    "\n",
    "search = arxiv.Search(\n",
    "    query=\"all:artificial intelligence AND all:climate change\",\n",
    "    max_results=1000,\n",
    "    sort_by=arxiv.SortCriterion.Relevance,\n",
    "    sort_order=arxiv.SortOrder.Descending,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d18c0f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Downloading metadata and fulltext of papers\n",
    "\n",
    "arxiv_papers = []\n",
    "unavailable = []\n",
    "\n",
    "for result in search.results():\n",
    "    temp = {}\n",
    "    \n",
    "    temp[\"entry_id\"] = result.entry_id\n",
    "    temp[\"updated\"] = result.updated\n",
    "    temp[\"published\"] = result.published\n",
    "    temp[\"title\"] = result.title\n",
    "    temp[\"authors\"] = result.authors\n",
    "    temp[\"summary\"] = result.summary.replace(\"\\n\", \" \")\n",
    "    temp[\"comment\"] = result.comment\n",
    "    temp[\"journal_ref\"] = result.journal_ref\n",
    "    temp[\"doi\"] = result.doi\n",
    "    temp[\"primary_category\"] = result.primary_category\n",
    "    temp[\"categories\"] = result.categories\n",
    "    temp[\"links\"] = result.links\n",
    "    temp[\"pdf_url\"] = result.pdf_url\n",
    "    temp[\"links\"] = result.links\n",
    "    \n",
    "    arxiv_papers.append(temp)\n",
    "    print(str(i), \".\", result.title)\n",
    "    \n",
    "    try:\n",
    "        result.download_pdf(dirpath=\"./data/arxiv/papers/\")\n",
    "        print(\"paper downloaded\\n\")\n",
    "    except:\n",
    "        print(\"-----> paper not available\\n\")\n",
    "        unavailable.append(result.title)\n",
    "        pass\n",
    "    \n",
    "print(len(unvailable), \"papers unavailable\")\n",
    "\n",
    "# Downloaded 231 papers automatically, 3 papers manually; log saved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0eef6a0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "unavailable\n",
    "\n",
    "# Downloaded these papers manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5cebfb48",
   "metadata": {},
   "outputs": [],
   "source": [
    "arxiv_papers[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "237a26df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving metadata for later use\n",
    "\n",
    "with open(\"./data/arxiv/metadata/arxiv.json\", \"w\") as arxiv_filename:\n",
    "    json.dump(arxiv_papers, arxiv_filename, default=str, indent=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10b16b91",
   "metadata": {},
   "source": [
    "## Metadata exploration\n",
    "\n",
    "- [Back to top](#arXiv-paper-collection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ec643b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "arxiv_df = pd.read_json(\"./data/arxiv/metadata/arxiv.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e8cda73",
   "metadata": {},
   "outputs": [],
   "source": [
    "arxiv_df\n",
    "\n",
    "# Since most of these papers are preprints, extensive exploration was skipped"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}