notebooks/Manuscript_Matcher_Data_Preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "DiVW55cvgeR4",
        "vU9eXAIxskCk",
        "32Grl6oDzvxB",
        "oKRV9U8B3cyK"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "### Setting Up"
      ],
      "metadata": {
        "id": "4LqG_dN9GBYC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%reload_ext autoreload\n",
        "%autoreload 2\n",
        "%matplotlib inline"
      ],
      "metadata": {
        "id": "_cExHjY4CKwD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "asVpledMCgiS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "P0YDHcgGClMp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd /content/drive/MyDrive/Manuscript-Matcher"
      ],
      "metadata": {
        "id": "D0ou8_yUCoXZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Pre-processing"
      ],
      "metadata": {
        "id": "Pi635UKhGGuJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Load CSV files"
      ],
      "metadata": {
        "id": "DiVW55cvgeR4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "journal_df = pd.read_csv(\"combined_journal_ranking_data.csv\")"
      ],
      "metadata": {
        "id": "IL-G8ndvFnlD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "articles_df = pd.read_csv(\"doaj_articles_data.csv\")"
      ],
      "metadata": {
        "id": "crUebGnSskoa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "asjc_df = pd.read_csv(\"asjc_codes.csv\")"
      ],
      "metadata": {
        "id": "Cst1I4hWe_Pd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Data cleaning"
      ],
      "metadata": {
        "id": "vU9eXAIxskCk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "journal_df = journal_df[['Title', 'Open Access', 'Best Quartile', 'ASJC Codes', 'ISO Language Codes']]\n",
        "journal_df.rename(columns={'Title': 'Journal Name'}, inplace=True)\n",
        "journal_df['Journal Name'] = journal_df['Journal Name'].apply(lambda x: x.upper())"
      ],
      "metadata": {
        "id": "4r4_F5DIqm4_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "articles_df = articles_df[['Journal Name', 'Title', 'Abstract']]\n",
        "articles_df['Journal Name'] = articles_df['Journal Name'].apply(lambda x: x.rsplit('(', 1)[0].strip())"
      ],
      "metadata": {
        "id": "DMvu5jCXrovp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "articles_df.duplicated().sum()"
      ],
      "metadata": {
        "id": "5fijJJXvqi5g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "articles_df.drop_duplicates(inplace=True)"
      ],
      "metadata": {
        "id": "AJjAXOOYq7Hc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "articles_df.isna().sum()"
      ],
      "metadata": {
        "id": "p80aTwC5q7KH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "articles_df.dropna(inplace=True)"
      ],
      "metadata": {
        "id": "Sm0cyWq4rSvX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "articles_df.shape"
      ],
      "metadata": {
        "id": "DOScMjkcq7Mq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Merge two datasets"
      ],
      "metadata": {
        "id": "32Grl6oDzvxB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = articles_df.merge(journal_df, on='Journal Name', how='left')"
      ],
      "metadata": {
        "id": "c8qVdxwq8KpQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "articles_df.shape, journal_df.shape, df.shape"
      ],
      "metadata": {
        "id": "5WFfQ8rk1BtU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Final data cleaning"
      ],
      "metadata": {
        "id": "oKRV9U8B3cyK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df.duplicated().sum()"
      ],
      "metadata": {
        "id": "6YWq2gmp1NFz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['Abstract'].duplicated().sum()"
      ],
      "metadata": {
        "id": "yg3FuoQ_-Z5h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['Title'].duplicated().sum()"
      ],
      "metadata": {
        "id": "dSGwLK_G-xq3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.drop_duplicates(inplace=True)\n",
        "df.drop_duplicates(subset=['Abstract'], inplace=True)\n",
        "df.drop_duplicates(subset=['Title'], inplace=True)"
      ],
      "metadata": {
        "id": "IlGp0uzaE4E0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.isna().sum()"
      ],
      "metadata": {
        "id": "IFN0Ld8pz88-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.dropna(inplace=True)"
      ],
      "metadata": {
        "id": "gzM9Ngmn1SFO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(df['Open Access'] == False).sum()"
      ],
      "metadata": {
        "id": "RLQoaax21bvu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(df['ISO Language Codes'] != 'ENG').sum()"
      ],
      "metadata": {
        "id": "TtvLdCiP2FJi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.drop(columns=['Open Access', 'Best Quartile', 'ISO Language Codes'], inplace=True)"
      ],
      "metadata": {
        "id": "Zc7R96EpWDJ7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.rename(columns={'Journal Name': 'journal', 'Title': 'title', 'Abstract': 'abstract', 'ASJC Codes': 'asjc_codes'}, inplace=True)"
      ],
      "metadata": {
        "id": "haDB-8X4VY8U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['journal'].value_counts()"
      ],
      "metadata": {
        "id": "Ij99NkFACBfN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.drop(df[df.journal == 'HELIYON'][1001:].index).reset_index(drop=True)"
      ],
      "metadata": {
        "id": "VSib6FgJCE1q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.drop(df[df.journal == 'SUSTAINABILITY'][1001:].index).reset_index(drop=True)"
      ],
      "metadata": {
        "id": "jpUSJXEmCFJX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.drop(df[df.journal == 'ENVIRONMENTAL RESEARCH LETTERS'][1001:].index).reset_index(drop=True)"
      ],
      "metadata": {
        "id": "V_24neZ4CFT4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "metadata": {
        "id": "3kEapLlW2Oye"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Save data"
      ],
      "metadata": {
        "id": "z2RUFw9P-CrA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df.info()"
      ],
      "metadata": {
        "id": "HiLoJPZv-QnW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.describe()"
      ],
      "metadata": {
        "id": "QU9Y_uN--VhU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "id": "PPQD1BkW-Ynh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.to_csv('/content/drive/MyDrive/Manuscript-Matcher/data.csv', index=False)"
      ],
      "metadata": {
        "id": "6eKlhvt7-GgV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Klb2ZwGk-dhu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}