BERT_Classification_Training_data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, csv\n",
    "from tqdm import tqdm, trange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file paths\n",
    "data_dir = \"/home/jack/Desktop/NN/clean/datasets/\"\n",
    "dataset = \"yelp\" # amazon / yelp / imagecaption\n",
    "train_0 = os.path.join(data_dir ,\"./{}/sentiment.train.0\".format(dataset))\n",
    "train_1 = os.path.join(data_dir,\"./{}/sentiment.train.1\".format(dataset))\n",
    "test_0 = os.path.join(data_dir,\"./{}/sentiment.test.0\".format(dataset))\n",
    "test_1 = os.path.join(data_dir,\"./{}/sentiment.test.1\".format(dataset))\n",
    "dev_0 = os.path.join(data_dir,\"./{}/sentiment.dev.0\".format(dataset))\n",
    "dev_1 = os.path.join(data_dir,\"./{}/sentiment.dev.1\".format(dataset))\n",
    "reference_0 = os.path.join(data_dir,\"./{}/reference.0\".format(dataset))\n",
    "reference_1 = os.path.join(data_dir,\"./{}/reference.1\".format(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_out = os.path.join(data_dir,\"{}/bert_classifier_training/train.csv\".format(dataset))\n",
    "dev_out = os.path.join(data_dir,\"{}/bert_classifier_training/dev.csv\".format(dataset))\n",
    "test_out = os.path.join(data_dir,\"{}/bert_classifier_training/test.csv\".format(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_classification_file(input_file_path_list, input_file_label_list, output_file_path):\n",
    "    \"\"\"\n",
    "    Create a csv file combining training data for BERT classification training.\n",
    "    input_file_path_list : Path of the input files\n",
    "    input_file_label_list : Correspoding labels for input_files\n",
    "    output_file_path : csv file path\n",
    "    \"\"\"\n",
    "    with open(output_file_path, \"w\") as out_fp:\n",
    "        writer = csv.writer(out_fp, delimiter=\"\\t\")\n",
    "        for i, file in enumerate(tqdm(input_file_path_list)):\n",
    "            with open(file) as fp:\n",
    "                for line in fp:\n",
    "                    writer.writerow([line.strip(),input_file_label_list[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:00<00:00,  4.88it/s]\n",
      "100%|██████████| 2/2 [00:00<00:00, 1645.47it/s]\n",
      "100%|██████████| 2/2 [00:00<00:00, 506.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "create_classification_file([train_0,train_1],[0,1], train_out)\n",
    "create_classification_file([test_0,test_1],[0,1], test_out)\n",
    "create_classification_file([dev_0,dev_1],[0,1], dev_out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('cleantransfer')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "1262644d422a82105623d802af89e6e9ed007275a479308d900e7081e9b63df8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}