08.corresponding-authors.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Created a merged corresponding authors dataset\n",
    "\n",
    "Identifies corresponding authors from PubMed Central author list if available. Otherwise, use PubMed author list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "import pandas\n",
    "import pubmedpy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare country assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>affiliation</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"Athena\" Research and Innovation Center, Athen...</td>\n",
       "      <td>GR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Athena\" Research and Innovation Center, Athen...</td>\n",
       "      <td>GR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"Claudio Munari\" Center for Epilepsy Surgery, ...</td>\n",
       "      <td>IT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"Momentum\" Membrane Protein Bioinformatics Res...</td>\n",
       "      <td>HU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"Momentum\" Membrane Protein Bioinformatics Res...</td>\n",
       "      <td>HU</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         affiliation country\n",
       "0  \"Athena\" Research and Innovation Center, Athen...      GR\n",
       "1  \"Athena\" Research and Innovation Center, Athen...      GR\n",
       "2  \"Claudio Munari\" Center for Epilepsy Surgery, ...      IT\n",
       "3  \"Momentum\" Membrane Protein Bioinformatics Res...      HU\n",
       "4  \"Momentum\" Membrane Protein Bioinformatics Res...      HU"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read affiliations to country mapping\n",
    "country_df = pandas.read_csv(\"data/affiliations/countries.tsv.xz\", sep='\\t', keep_default_na=False)\n",
    "# be careful reading country codes: Namibia code is NA\n",
    "assert country_df.notna().all(axis=None)\n",
    "country_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "source = 'pubmed'\n",
    "directory = pathlib.Path('data') / source\n",
    "affil_df = pandas.read_csv(directory / \"affiliations.tsv.xz\", sep=\"\\t\")\n",
    "affil_df = affil_df.merge(country_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_authors(source: str) -> pandas.DataFrame:\n",
    "    \"\"\"\n",
    "    Read authors table for source, and add a countries column.\n",
    "    \"\"\"\n",
    "    directory = pathlib.Path('data') / source\n",
    "    # read authors\n",
    "    author_df = pandas.read_csv(directory / \"authors.tsv.xz\", sep=\"\\t\")\n",
    "    # read author affiliations\n",
    "    affil_df = pandas.read_csv(directory / \"affiliations.tsv.xz\", sep=\"\\t\")\n",
    "    affil_df = affil_df.merge(country_df)\n",
    "    # assign countries to authors by affiliations\n",
    "    primary_key = {\"pubmed\": \"pmid\", \"pmc\": \"pmcid\"}[source]\n",
    "    countries_df = (\n",
    "        affil_df\n",
    "        .groupby([primary_key, \"position\"])\n",
    "        .country\n",
    "        .apply(lambda x: \",\".join(sorted(set(x))))\n",
    "        .reset_index()\n",
    "        .rename(columns={\"country\": \"countries\"})\n",
    "    )\n",
    "    author_df = author_df.merge(countries_df, how='left')\n",
    "    return author_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read pubmed authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>doi</th>\n",
       "      <th>journal</th>\n",
       "      <th>publication_date</th>\n",
       "      <th>pmc_cited_by_count</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1038/378516a0</td>\n",
       "      <td>Nature</td>\n",
       "      <td>1995-11-30</td>\n",
       "      <td>2</td>\n",
       "      <td>Mapping the genome one molecule at a time--opt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7479891</td>\n",
       "      <td>PMC40523</td>\n",
       "      <td>10.1073/pnas.92.24.10821</td>\n",
       "      <td>Proc Natl Acad Sci U S A</td>\n",
       "      <td>1995-11-21</td>\n",
       "      <td>4</td>\n",
       "      <td>Ahead of schedule and under budget: the Genome...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid     pmcid                       doi                   journal  \\\n",
       "0  7477412       NaN          10.1038/378516a0                    Nature   \n",
       "1  7479891  PMC40523  10.1073/pnas.92.24.10821  Proc Natl Acad Sci U S A   \n",
       "\n",
       "  publication_date  pmc_cited_by_count  \\\n",
       "0       1995-11-30                   2   \n",
       "1       1995-11-21                   4   \n",
       "\n",
       "                                               title  \n",
       "0  Mapping the genome one molecule at a time--opt...  \n",
       "1  Ahead of schedule and under budget: the Genome...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubmed_df = pandas.read_csv(\"data/pubmed/articles.tsv.xz\", sep=\"\\t\")\n",
    "pubmed_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>doi</th>\n",
       "      <th>journal</th>\n",
       "      <th>publication_date</th>\n",
       "      <th>pmc_cited_by_count</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1038/378516a0</td>\n",
       "      <td>Nature</td>\n",
       "      <td>1995-11-30</td>\n",
       "      <td>2</td>\n",
       "      <td>Mapping the genome one molecule at a time--opt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7479891</td>\n",
       "      <td>PMC40523</td>\n",
       "      <td>10.1073/pnas.92.24.10821</td>\n",
       "      <td>Proc Natl Acad Sci U S A</td>\n",
       "      <td>1995-11-21</td>\n",
       "      <td>4</td>\n",
       "      <td>Ahead of schedule and under budget: the Genome...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7479895</td>\n",
       "      <td>PMC40527</td>\n",
       "      <td>10.1073/pnas.92.24.10841</td>\n",
       "      <td>Proc Natl Acad Sci U S A</td>\n",
       "      <td>1995-11-21</td>\n",
       "      <td>11</td>\n",
       "      <td>How is the Human Genome Project doing, and wha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7497116</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1089/cmb.1995.2.139</td>\n",
       "      <td>J Comput Biol</td>\n",
       "      <td>1995</td>\n",
       "      <td>1</td>\n",
       "      <td>Four strikes against physical mapping of DNA.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7497128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1089/cmb.1995.2.219</td>\n",
       "      <td>J Comput Biol</td>\n",
       "      <td>1995</td>\n",
       "      <td>2</td>\n",
       "      <td>Physical mapping by STS hybridization: algorit...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid     pmcid                       doi                   journal  \\\n",
       "0  7477412       NaN          10.1038/378516a0                    Nature   \n",
       "1  7479891  PMC40523  10.1073/pnas.92.24.10821  Proc Natl Acad Sci U S A   \n",
       "2  7479895  PMC40527  10.1073/pnas.92.24.10841  Proc Natl Acad Sci U S A   \n",
       "3  7497116       NaN    10.1089/cmb.1995.2.139             J Comput Biol   \n",
       "4  7497128       NaN    10.1089/cmb.1995.2.219             J Comput Biol   \n",
       "\n",
       "  publication_date  pmc_cited_by_count  \\\n",
       "0       1995-11-30                   2   \n",
       "1       1995-11-21                   4   \n",
       "2       1995-11-21                  11   \n",
       "3             1995                   1   \n",
       "4             1995                   2   \n",
       "\n",
       "                                               title  \n",
       "0  Mapping the genome one molecule at a time--opt...  \n",
       "1  Ahead of schedule and under budget: the Genome...  \n",
       "2  How is the Human Genome Project doing, and wha...  \n",
       "3      Four strikes against physical mapping of DNA.  \n",
       "4  Physical mapping by STS hybridization: algorit...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubmed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>position</th>\n",
       "      <th>fore_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>reverse_position</th>\n",
       "      <th>countries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>1</td>\n",
       "      <td>A H</td>\n",
       "      <td>Samad</td>\n",
       "      <td>10</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7477412</td>\n",
       "      <td>2</td>\n",
       "      <td>W W</td>\n",
       "      <td>Cai</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid  position fore_name last_name  reverse_position countries\n",
       "0  7477412         1       A H     Samad                10        US\n",
       "1  7477412         2       W W       Cai                 9       NaN"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubmed_author_df = read_authors(\"pubmed\")\n",
    "pubmed_author_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pmc_author_df = read_authors(\"pmc\")\n",
    "# pmc_author_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def get_corresponding(df):\n",
    "# #     if df.corresponding.any():\n",
    "# #         df = df.query(\"corresponding == 1\")\n",
    "# #         df['use_last'] = 0\n",
    "# #     else:\n",
    "#     df = df.query(\"reverse_position == 1\")\n",
    "#     df['use_last'] = 1\n",
    "#     return df.assign(n_corresponding=len(df))\n",
    "\n",
    "# pmc_corresp_df = (\n",
    "#     pubmed_df[['pmid', 'pmcid']]\n",
    "#     .merge(pmc_author_df)\n",
    "#     .assign(source=\"pmc\")\n",
    "#     .groupby('pmid')\n",
    "#     .apply(get_corresponding)\n",
    "# )\n",
    "# pmc_corresp_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pmc_corresp_df.countries.notna().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>position</th>\n",
       "      <th>fore_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>reverse_position</th>\n",
       "      <th>countries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7477412</td>\n",
       "      <td>10</td>\n",
       "      <td>B</td>\n",
       "      <td>Porter</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>7479891</td>\n",
       "      <td>1</td>\n",
       "      <td>F S</td>\n",
       "      <td>Collins</td>\n",
       "      <td>1</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>7479895</td>\n",
       "      <td>2</td>\n",
       "      <td>F S</td>\n",
       "      <td>Collins</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>7497116</td>\n",
       "      <td>4</td>\n",
       "      <td>R</td>\n",
       "      <td>Shamir</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>7497128</td>\n",
       "      <td>2</td>\n",
       "      <td>S</td>\n",
       "      <td>Istrail</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1103509</th>\n",
       "      <td>32612807</td>\n",
       "      <td>5</td>\n",
       "      <td>Tarcisio</td>\n",
       "      <td>Mendes de Farias</td>\n",
       "      <td>1</td>\n",
       "      <td>CH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1103517</th>\n",
       "      <td>32760576</td>\n",
       "      <td>8</td>\n",
       "      <td>Matthew</td>\n",
       "      <td>McAuliffe</td>\n",
       "      <td>1</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1103521</th>\n",
       "      <td>32913631</td>\n",
       "      <td>4</td>\n",
       "      <td>Joshua W K</td>\n",
       "      <td>Ho</td>\n",
       "      <td>1</td>\n",
       "      <td>AU,HK,US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1103527</th>\n",
       "      <td>33016592</td>\n",
       "      <td>6</td>\n",
       "      <td>Charles</td>\n",
       "      <td>Chen</td>\n",
       "      <td>1</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1103532</th>\n",
       "      <td>33016595</td>\n",
       "      <td>5</td>\n",
       "      <td>Lucía</td>\n",
       "      <td>Gutiérrez</td>\n",
       "      <td>1</td>\n",
       "      <td>US,UY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>179487 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             pmid  position   fore_name         last_name  reverse_position  \\\n",
       "9         7477412        10           B            Porter                 1   \n",
       "10        7479891         1         F S           Collins                 1   \n",
       "12        7479895         2         F S           Collins                 1   \n",
       "16        7497116         4           R            Shamir                 1   \n",
       "18        7497128         2           S           Istrail                 1   \n",
       "...           ...       ...         ...               ...               ...   \n",
       "1103509  32612807         5    Tarcisio  Mendes de Farias                 1   \n",
       "1103517  32760576         8     Matthew         McAuliffe                 1   \n",
       "1103521  32913631         4  Joshua W K                Ho                 1   \n",
       "1103527  33016592         6     Charles              Chen                 1   \n",
       "1103532  33016595         5       Lucía         Gutiérrez                 1   \n",
       "\n",
       "        countries  \n",
       "9             NaN  \n",
       "10             US  \n",
       "12            NaN  \n",
       "16            NaN  \n",
       "18            NaN  \n",
       "...           ...  \n",
       "1103509        CH  \n",
       "1103517        US  \n",
       "1103521  AU,HK,US  \n",
       "1103527        US  \n",
       "1103532     US,UY  \n",
       "\n",
       "[179487 rows x 6 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pubmed_author_df.query('reverse_position == 1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pubmed_corresp_df.countries.notna().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>position</th>\n",
       "      <th>fore_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>reverse_position</th>\n",
       "      <th>countries</th>\n",
       "      <th>source</th>\n",
       "      <th>use_last</th>\n",
       "      <th>fore_name_simple</th>\n",
       "      <th>last_name_simple</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>10</td>\n",
       "      <td>B</td>\n",
       "      <td>Porter</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>porter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7479891</td>\n",
       "      <td>1</td>\n",
       "      <td>F S</td>\n",
       "      <td>Collins</td>\n",
       "      <td>1</td>\n",
       "      <td>US</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>collins</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid  position fore_name last_name  reverse_position countries  source  \\\n",
       "0  7477412        10         B    Porter                 1       NaN  pubmed   \n",
       "1  7479891         1       F S   Collins                 1        US  pubmed   \n",
       "\n",
       "   use_last fore_name_simple last_name_simple  \n",
       "0         1                            porter  \n",
       "1         1                           collins  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# corresp_df = pandas.concat([pmc_corresp_df, pubmed_corresp_df], sort=False).reset_index(drop=True)\n",
    "corresp_df = (pubmed_author_df\n",
    "    .query('reverse_position == 1')\n",
    "    .assign(source='pubmed', use_last = 1))\n",
    "fore_df = pandas.read_csv(\"data/names/fore-names.tsv.xz\", sep=\"\\t\", keep_default_na=False)\n",
    "last_df = pandas.read_csv(\"data/names/last-names.tsv.xz\", sep=\"\\t\", keep_default_na=False)\n",
    "corresp_df = (\n",
    "    corresp_df\n",
    "    .merge(fore_df[[\"fore_name\", \"fore_name_simple\"]], how=\"left\")\n",
    "    .merge(last_df[[\"last_name\", \"last_name_simple\"]], how=\"left\")\n",
    ")\n",
    "corresp_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of authors with 1+ assinged countries by source\n",
    "# False indicates no assigned countries.\n",
    "# True indicates one or more countries\n",
    "tab = pandas.crosstab(\n",
    "    corresp_df.source,\n",
    "    corresp_df.countries.notnull(),\n",
    "    margins=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>countries</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>pubmed</th>\n",
       "      <td>52.6%</td>\n",
       "      <td>47.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>52.6%</td>\n",
       "      <td>47.4%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "countries  False  True \n",
       "source                 \n",
       "pubmed     52.6%  47.4%\n",
       "All        52.6%  47.4%"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# percent of authors with 1+ assinged countries by source\n",
    "pandas.crosstab(\n",
    "    corresp_df.source,\n",
    "    corresp_df.countries.notnull(),\n",
    "    margins=True, normalize=\"index\"\n",
    ").applymap(\"{:.1%}\".format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>doi</th>\n",
       "      <th>journal</th>\n",
       "      <th>publication_date</th>\n",
       "      <th>pmc_cited_by_count</th>\n",
       "      <th>title</th>\n",
       "      <th>source</th>\n",
       "      <th>countries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1038/378516a0</td>\n",
       "      <td>Nature</td>\n",
       "      <td>1995-11-30</td>\n",
       "      <td>2</td>\n",
       "      <td>Mapping the genome one molecule at a time--opt...</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7479891</td>\n",
       "      <td>PMC40523</td>\n",
       "      <td>10.1073/pnas.92.24.10821</td>\n",
       "      <td>Proc Natl Acad Sci U S A</td>\n",
       "      <td>1995-11-21</td>\n",
       "      <td>4</td>\n",
       "      <td>Ahead of schedule and under budget: the Genome...</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>US</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7479895</td>\n",
       "      <td>PMC40527</td>\n",
       "      <td>10.1073/pnas.92.24.10841</td>\n",
       "      <td>Proc Natl Acad Sci U S A</td>\n",
       "      <td>1995-11-21</td>\n",
       "      <td>11</td>\n",
       "      <td>How is the Human Genome Project doing, and wha...</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7497116</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1089/cmb.1995.2.139</td>\n",
       "      <td>J Comput Biol</td>\n",
       "      <td>1995</td>\n",
       "      <td>1</td>\n",
       "      <td>Four strikes against physical mapping of DNA.</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7497128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1089/cmb.1995.2.219</td>\n",
       "      <td>J Comput Biol</td>\n",
       "      <td>1995</td>\n",
       "      <td>2</td>\n",
       "      <td>Physical mapping by STS hybridization: algorit...</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid     pmcid                       doi                   journal  \\\n",
       "0  7477412       NaN          10.1038/378516a0                    Nature   \n",
       "1  7479891  PMC40523  10.1073/pnas.92.24.10821  Proc Natl Acad Sci U S A   \n",
       "2  7479895  PMC40527  10.1073/pnas.92.24.10841  Proc Natl Acad Sci U S A   \n",
       "3  7497116       NaN    10.1089/cmb.1995.2.139             J Comput Biol   \n",
       "4  7497128       NaN    10.1089/cmb.1995.2.219             J Comput Biol   \n",
       "\n",
       "  publication_date  pmc_cited_by_count  \\\n",
       "0       1995-11-30                   2   \n",
       "1       1995-11-21                   4   \n",
       "2       1995-11-21                  11   \n",
       "3             1995                   1   \n",
       "4             1995                   2   \n",
       "\n",
       "                                               title  source countries  \n",
       "0  Mapping the genome one molecule at a time--opt...  pubmed       NaN  \n",
       "1  Ahead of schedule and under budget: the Genome...  pubmed        US  \n",
       "2  How is the Human Genome Project doing, and wha...  pubmed       NaN  \n",
       "3      Four strikes against physical mapping of DNA.  pubmed       NaN  \n",
       "4  Physical mapping by STS hybridization: algorit...  pubmed       NaN  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pmc_country = (\n",
    "    pubmed_df\n",
    "    .merge(corresp_df[['pmid', 'source', 'countries']].drop_duplicates())\n",
    ")\n",
    "pmc_country.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If available, collect PMCID of articles with no countries to query more affiliations from:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "pmcids_query = (pmc_country.query('(countries != countries) and (pmcid == pmcid)'))['pmcid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2           PMC40527\n",
       "20          PMC41521\n",
       "28         PMC225990\n",
       "48        PMC1336852\n",
       "67          PMC44177\n",
       "             ...    \n",
       "179332    PMC7184535\n",
       "179388    PMC6986235\n",
       "179391    PMC6986237\n",
       "179394    PMC7390749\n",
       "179454    PMC6426537\n",
       "Name: pmcid, Length: 37646, dtype: object"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pmcids_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_frontmatter_etree_via_api(pmcid):\n",
    "    url = f\"https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi?verb=GetRecord&identifier=oai:pubmedcentral.nih.gov:{pmcid[3:]}&metadataPrefix=pmc_fm\"\n",
    "    response = requests.get(url)\n",
    "    tree = etree.fromstring(response.content)\n",
    "    article = tree.find(\"{*}GetRecord/{*}record/{*}metadata/{*}article\")\n",
    "    return article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from pubmedpy.pmc_oai import extract_authors_from_article\n",
    "# from pubmedpy.tests.test_pmc_oai import get_frontmatter_etree_via_api\n",
    "from lxml import etree\n",
    "# from lxml.etree import tostring\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'pmcid': 'PMC6986237',\n",
       "  'position': 1,\n",
       "  'fore_name': 'Megan E.',\n",
       "  'last_name': 'Barefoot',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 6,\n",
       "  'affiliations': []},\n",
       " {'pmcid': 'PMC6986237',\n",
       "  'position': 2,\n",
       "  'fore_name': 'Rency S.',\n",
       "  'last_name': 'Varghese',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 5,\n",
       "  'affiliations': []},\n",
       " {'pmcid': 'PMC6986237',\n",
       "  'position': 3,\n",
       "  'fore_name': 'Yuan',\n",
       "  'last_name': 'Zhou',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 4,\n",
       "  'affiliations': []},\n",
       " {'pmcid': 'PMC6986237',\n",
       "  'position': 4,\n",
       "  'fore_name': 'Cristina',\n",
       "  'last_name': 'Di Poto',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 3,\n",
       "  'affiliations': []},\n",
       " {'pmcid': 'PMC6986237',\n",
       "  'position': 5,\n",
       "  'fore_name': 'Alessia',\n",
       "  'last_name': 'Ferrarini',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 2,\n",
       "  'affiliations': []},\n",
       " {'pmcid': 'PMC6986237',\n",
       "  'position': 6,\n",
       "  'fore_name': 'Habtom W.',\n",
       "  'last_name': 'Ressom',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 1,\n",
       "  'affiliations': []}]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "art = get_frontmatter_etree_via_api('PMC6986237')\n",
    "extract_authors_from_article(art)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<article xmlns=\"https://jats.nlm.nih.gov/ns/archiving/1.2/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns:ali=\"http://www.niso.org/schemas/ali/1.0/\" xmlns:mml=\"http://www.w3.org/1998/Math/MathML\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xsi:schemaLocation=\"https://jats.nlm.nih.gov/ns/archiving/1.2/ https://jats.nlm.nih.gov/archiving/1.2/xsd/JATS-archivearticle1.xsd\" article-type=\"research-article\">\n",
      "  <front>\n",
      "    <journal-meta>\n",
      "      <journal-id journal-id-type=\"nlm-journal-id\">101243413</journal-id>\n",
      "      <journal-id journal-id-type=\"pubmed-jr-id\">32722</journal-id>\n",
      "      <journal-id journal-id-type=\"nlm-ta\">Conf Proc IEEE Eng Med Biol Soc</journal-id>\n",
      "      <journal-id journal-id-type=\"iso-abbrev\">Conf Proc IEEE Eng Med Biol Soc</journal-id>\n",
      "      <journal-title-group>\n",
      "        <journal-title>Conference proceedings : ... Annual International Conference of the IEEE Engineering in Medicine and Biology Society. IEEE Engineering in Medicine and Biology Society. Annual Conference</journal-title>\n",
      "      </journal-title-group>\n",
      "      <issn pub-type=\"ppub\">1557-170X</issn>\n",
      "    </journal-meta>\n",
      "    <article-meta>\n",
      "      <article-id pub-id-type=\"accession\">PMC6986235</article-id>\n",
      "      <article-id pub-id-type=\"pmcid\">PMC6986235</article-id>\n",
      "      <article-id pub-id-type=\"pmc-uid\">6986235</article-id>\n",
      "      <article-id pub-id-type=\"pmid\">31945887</article-id>\n",
      "      <article-id pub-id-type=\"pmid\">31945887</article-id>\n",
      "      <article-id pub-id-type=\"doi\">10.1109/EMBC.2019.8857049</article-id>\n",
      "      <article-id pub-id-type=\"manuscript\">nihpa1026053</article-id>\n",
      "      <article-categories>\n",
      "        <subj-group subj-group-type=\"heading\">\n",
      "          <subject>Article</subject>\n",
      "        </subj-group>\n",
      "      </article-categories>\n",
      "      <title-group>\n",
      "        <article-title>MOTA: Multi-omic integrative analysis for biomarker discovery</article-title>\n",
      "      </title-group>\n",
      "      <contrib-group>\n",
      "        <contrib contrib-type=\"author\">\n",
      "          <name>\n",
      "            <surname>Fan</surname>\n",
      "            <given-names>Ziling</given-names>\n",
      "          </name>\n",
      "          <aff id=\"A1\">Department of Biochemistry and Molecular and Cellular Biology, Georgetown University Medical Center, Washington, DC</aff>\n",
      "        </contrib>\n",
      "      </contrib-group>\n",
      "      <contrib-group>\n",
      "        <contrib contrib-type=\"author\">\n",
      "          <name>\n",
      "            <surname>Zhou</surname>\n",
      "            <given-names>Yuan</given-names>\n",
      "          </name>\n",
      "        </contrib>\n",
      "        <contrib contrib-type=\"author\">\n",
      "          <name>\n",
      "            <surname>Ressom</surname>\n",
      "            <given-names>Habtom W.</given-names>\n",
      "          </name>\n",
      "          <aff id=\"A2\">Department of Oncology, Lombardi Comprehensive Cancer Center, Georgetown University Medical Center, Washington, DC</aff>\n",
      "        </contrib>\n",
      "      </contrib-group>\n",
      "      <author-notes>\n",
      "        <corresp id=\"CR1\">Corresponding author: <email>hwr@georgetown.edu</email></corresp>\n",
      "      </author-notes>\n",
      "      <pub-date pub-type=\"nihms-submitted\">\n",
      "        <day>30</day>\n",
      "        <month>4</month>\n",
      "        <year>2019</year>\n",
      "      </pub-date>\n",
      "      <pub-date pub-type=\"ppub\">\n",
      "        <month>7</month>\n",
      "        <year>2019</year>\n",
      "      </pub-date>\n",
      "      <pub-date pub-type=\"pmc-release\">\n",
      "        <day>28</day>\n",
      "        <month>1</month>\n",
      "        <year>2020</year>\n",
      "      </pub-date>\n",
      "      <volume>2019</volume>\n",
      "      <fpage>243</fpage>\n",
      "      <lpage>247</lpage>\n",
      "      <!--elocation-id from pubmed: 10.1109/EMBC.2019.8857049-->\n",
      "      <abstract id=\"ABS1\">\n",
      "        <p id=\"P1\">Recent advancement of omic technologies provides researchers with opportunities to search for disease biomarkers at the systems level. However, selection of biomarker candidates from a large number of molecules involved at various layers of the biological system is challenging. In this paper, we propose multi-omic integrative analysis (MOTA), a network-based method that uses information from multi-omic data to identify candidate disease biomarkers. We evaluated the performance of MOTA in selecting disease-associated molecules from four sets of multi-omic data representing three cohorts of hepatocellular carcinoma (HCC) cases and patients with liver cirrhosis. The results demonstrate that MOTA leads to selection of more biomarker candidates that shared by two different cohorts compared to traditional statistical methods. Also, the networks constructed by MOTA allow users to investigate biological significance of the selected biomarker candidates.</p>\n",
      "      </abstract>\n",
      "      <kwd-group>\n",
      "        <kwd>multi-omic</kwd>\n",
      "        <kwd>network analysis</kwd>\n",
      "        <kwd>feature selection</kwd>\n",
      "        <kwd>liver cancer</kwd>\n",
      "        <kwd>liver cirrhosis</kwd>\n",
      "        <kwd>biomarkers</kwd>\n",
      "      </kwd-group>\n",
      "    </article-meta>\n",
      "  </front>\n",
      "</article>\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "art = get_frontmatter_etree_via_api('PMC6986235')\n",
    "print(etree.tostring(art, encoding = 'unicode'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'pmcid': 'PMC6986235',\n",
       "  'position': 1,\n",
       "  'fore_name': 'Ziling',\n",
       "  'last_name': 'Fan',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 3,\n",
       "  'affiliations': []},\n",
       " {'pmcid': 'PMC6986235',\n",
       "  'position': 2,\n",
       "  'fore_name': 'Yuan',\n",
       "  'last_name': 'Zhou',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 2,\n",
       "  'affiliations': []},\n",
       " {'pmcid': 'PMC6986235',\n",
       "  'position': 3,\n",
       "  'fore_name': 'Habtom W.',\n",
       "  'last_name': 'Ressom',\n",
       "  'corresponding': 0,\n",
       "  'reverse_position': 1,\n",
       "  'affiliations': []}]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_authors_from_article(art)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analyze sources for corresponding authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pmid</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>doi</th>\n",
       "      <th>journal</th>\n",
       "      <th>publication_date</th>\n",
       "      <th>pmc_cited_by_count</th>\n",
       "      <th>title</th>\n",
       "      <th>source</th>\n",
       "      <th>use_last</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7477412</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1038/378516a0</td>\n",
       "      <td>Nature</td>\n",
       "      <td>1995-11-30</td>\n",
       "      <td>2</td>\n",
       "      <td>Mapping the genome one molecule at a time--opt...</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7479891</td>\n",
       "      <td>PMC40523</td>\n",
       "      <td>10.1073/pnas.92.24.10821</td>\n",
       "      <td>Proc Natl Acad Sci U S A</td>\n",
       "      <td>1995-11-21</td>\n",
       "      <td>4</td>\n",
       "      <td>Ahead of schedule and under budget: the Genome...</td>\n",
       "      <td>pubmed</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      pmid     pmcid                       doi                   journal  \\\n",
       "0  7477412       NaN          10.1038/378516a0                    Nature   \n",
       "1  7479891  PMC40523  10.1073/pnas.92.24.10821  Proc Natl Acad Sci U S A   \n",
       "\n",
       "  publication_date  pmc_cited_by_count  \\\n",
       "0       1995-11-30                   2   \n",
       "1       1995-11-21                   4   \n",
       "\n",
       "                                               title  source  use_last  \n",
       "0  Mapping the genome one molecule at a time--opt...  pubmed         1  \n",
       "1  Ahead of schedule and under budget: the Genome...  pubmed         1  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_df = (\n",
    "    pubmed_df\n",
    "    .merge(corresp_df[['pmid', 'source', 'use_last']].drop_duplicates())\n",
    ")\n",
    "source_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>source</th>\n",
       "      <th>pubmed</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>journal</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A A Case Rep</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AACN Adv Crit Care</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AACN Clin Issues</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAOHN J</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAPS J</th>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zygote</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eNeuro</th>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mBio</th>\n",
       "      <td>204</td>\n",
       "      <td>204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mSphere</th>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>179487</td>\n",
       "      <td>179487</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4290 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "source              pubmed     All\n",
       "journal                           \n",
       "A A Case Rep             1       1\n",
       "AACN Adv Crit Care       6       6\n",
       "AACN Clin Issues         2       2\n",
       "AAOHN J                  1       1\n",
       "AAPS J                  29      29\n",
       "...                    ...     ...\n",
       "Zygote                   2       2\n",
       "eNeuro                   8       8\n",
       "mBio                   204     204\n",
       "mSphere                 37      37\n",
       "All                 179487  179487\n",
       "\n",
       "[4290 rows x 2 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of articles by authorship source by journal\n",
    "pandas.crosstab(source_df.journal, source_df.source, margins=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>source</th>\n",
       "      <th>pubmed</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>publication_date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1993</th>\n",
       "      <td>84</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994</th>\n",
       "      <td>67</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>63</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>150</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1997</th>\n",
       "      <td>135</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>327</td>\n",
       "      <td>327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1999</th>\n",
       "      <td>349</td>\n",
       "      <td>349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2000</th>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2001</th>\n",
       "      <td>1209</td>\n",
       "      <td>1209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2002</th>\n",
       "      <td>1849</td>\n",
       "      <td>1849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2003</th>\n",
       "      <td>2708</td>\n",
       "      <td>2708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2004</th>\n",
       "      <td>4026</td>\n",
       "      <td>4026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2005</th>\n",
       "      <td>5373</td>\n",
       "      <td>5373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2006</th>\n",
       "      <td>6058</td>\n",
       "      <td>6058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2007</th>\n",
       "      <td>6335</td>\n",
       "      <td>6335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2008</th>\n",
       "      <td>7224</td>\n",
       "      <td>7224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009</th>\n",
       "      <td>9029</td>\n",
       "      <td>9029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2010</th>\n",
       "      <td>9790</td>\n",
       "      <td>9790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011</th>\n",
       "      <td>10761</td>\n",
       "      <td>10761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012</th>\n",
       "      <td>11497</td>\n",
       "      <td>11497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013</th>\n",
       "      <td>11889</td>\n",
       "      <td>11889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014</th>\n",
       "      <td>12840</td>\n",
       "      <td>12840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015</th>\n",
       "      <td>13404</td>\n",
       "      <td>13404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016</th>\n",
       "      <td>14067</td>\n",
       "      <td>14067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017</th>\n",
       "      <td>15182</td>\n",
       "      <td>15182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018</th>\n",
       "      <td>15707</td>\n",
       "      <td>15707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019</th>\n",
       "      <td>15770</td>\n",
       "      <td>15770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020</th>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>176624</td>\n",
       "      <td>176624</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "source            pubmed     All\n",
       "publication_date                \n",
       "1993                  84      84\n",
       "1994                  67      67\n",
       "1995                  63      63\n",
       "1996                 150     150\n",
       "1997                 135     135\n",
       "1998                 327     327\n",
       "1999                 349     349\n",
       "2000                 716     716\n",
       "2001                1209    1209\n",
       "2002                1849    1849\n",
       "2003                2708    2708\n",
       "2004                4026    4026\n",
       "2005                5373    5373\n",
       "2006                6058    6058\n",
       "2007                6335    6335\n",
       "2008                7224    7224\n",
       "2009                9029    9029\n",
       "2010                9790    9790\n",
       "2011               10761   10761\n",
       "2012               11497   11497\n",
       "2013               11889   11889\n",
       "2014               12840   12840\n",
       "2015               13404   13404\n",
       "2016               14067   14067\n",
       "2017               15182   15182\n",
       "2018               15707   15707\n",
       "2019               15770   15770\n",
       "2020                  15      15\n",
       "All               176624  176624"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of articles by authorship source by year\n",
    "pandas.crosstab(source_df.publication_date.str.slice(0, 4), source_df.source, margins=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>corresp_has_countries</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>journal</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A A Case Rep</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AACN Adv Crit Care</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AACN Clin Issues</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAOHN J</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAPS J</th>\n",
       "      <td>21</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zygote</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eNeuro</th>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mBio</th>\n",
       "      <td>45</td>\n",
       "      <td>159</td>\n",
       "      <td>204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mSphere</th>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>94443</td>\n",
       "      <td>85044</td>\n",
       "      <td>179487</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4290 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "corresp_has_countries  False   True     All\n",
       "journal                                    \n",
       "A A Case Rep               1      0       1\n",
       "AACN Adv Crit Care         0      6       6\n",
       "AACN Clin Issues           2      0       2\n",
       "AAOHN J                    0      1       1\n",
       "AAPS J                    21      8      29\n",
       "...                      ...    ...     ...\n",
       "Zygote                     1      1       2\n",
       "eNeuro                     0      8       8\n",
       "mBio                      45    159     204\n",
       "mSphere                    0     37      37\n",
       "All                    94443  85044  179487\n",
       "\n",
       "[4290 rows x 3 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of articles where all corresponding authors have assigned countries by journal\n",
    "articles_with_na_countries = set(corresp_df.loc[corresp_df.countries.isna()].pmid)\n",
    "source_df[\"corresp_has_countries\"] = ~source_df.pmid.isin(articles_with_na_countries)\n",
    "pandas.crosstab(source_df.journal, source_df.corresp_has_countries, margins=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>corresp_has_countries</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>journal</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A A Case Rep</th>\n",
       "      <td>100.0%</td>\n",
       "      <td>0.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AACN Adv Crit Care</th>\n",
       "      <td>0.0%</td>\n",
       "      <td>100.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AACN Clin Issues</th>\n",
       "      <td>100.0%</td>\n",
       "      <td>0.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAOHN J</th>\n",
       "      <td>0.0%</td>\n",
       "      <td>100.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AAPS J</th>\n",
       "      <td>72.4%</td>\n",
       "      <td>27.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Zygote</th>\n",
       "      <td>50.0%</td>\n",
       "      <td>50.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eNeuro</th>\n",
       "      <td>0.0%</td>\n",
       "      <td>100.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mBio</th>\n",
       "      <td>22.1%</td>\n",
       "      <td>77.9%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mSphere</th>\n",
       "      <td>0.0%</td>\n",
       "      <td>100.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>52.6%</td>\n",
       "      <td>47.4%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4290 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "corresp_has_countries   False   True \n",
       "journal                              \n",
       "A A Case Rep           100.0%    0.0%\n",
       "AACN Adv Crit Care       0.0%  100.0%\n",
       "AACN Clin Issues       100.0%    0.0%\n",
       "AAOHN J                  0.0%  100.0%\n",
       "AAPS J                  72.4%   27.6%\n",
       "...                       ...     ...\n",
       "Zygote                  50.0%   50.0%\n",
       "eNeuro                   0.0%  100.0%\n",
       "mBio                    22.1%   77.9%\n",
       "mSphere                  0.0%  100.0%\n",
       "All                     52.6%   47.4%\n",
       "\n",
       "[4290 rows x 2 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Percent of articles where all corresponding authors have assigned countries by journal\n",
    "pandas.crosstab(\n",
    "    source_df.journal, source_df.corresp_has_countries,\n",
    "    margins=True, normalize=\"index\"\n",
    ").applymap(\"{:.1%}\".format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "corresp_df.to_csv('data/names/corresponding-authors.tsv.xz', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}