Assignment 3/21CS30065_Assignment_3_Part_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "🔠 Natural Language Processing (CS60075) Autumn 2024, IIT Kharagpur\n",
    "\n",
    "📃 Assignment 3 Part 2: [Hate Speech Classification using Few-shot Prompting](https://sites.google.com/view/nlp-cs-iit-kgp/assignments)\n",
    "\n",
    "👦🏻 Author: [Prasanna Paithankar (21CS30065)](https://cse.iitkgp.ac.in/~prasannabp/)\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 📚 Import Libraries and Resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Codes\\Natural-Language-Processing-Aut-24\\venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From d:\\Codes\\Natural-Language-Processing-Aut-24\\venv\\Lib\\site-packages\\tf_keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from datasets import Dataset\n",
    "from dotenv import load_dotenv\n",
    "from huggingface_hub import HfFolder, login\n",
    "from sklearn.metrics import (classification_report,\n",
    "                             precision_recall_fscore_support)\n",
    "from transformers import (AutoConfig, AutoModelForSequenceClassification,\n",
    "                          AutoTokenizer, Trainer, TrainingArguments)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 🤗 Load Hugging Face Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
      "Token is valid (permission: fineGrained).\n",
      "Your token has been saved to C:\\Users\\paith\\.cache\\huggingface\\token\n",
      "Login successful\n"
     ]
    }
   ],
   "source": [
    "load_dotenv()\n",
    "HUGGING_FACE_TOKEN = os.getenv(\"HUGGING_FACE_TOKEN\")\n",
    "login(token=HUGGING_FACE_TOKEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 🧪 Load & Preprocess Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "iddict = {\"normal\": 0, \"hatespeech\": 1, \"offensive\": 2}\n",
    "labeldict = {0: \"normal\", 1: \"hatespeech\", 2: \"offensive\"}\n",
    "\n",
    "train_data = pd.read_csv('dataset/NLP_ass_train.tsv',\n",
    "                         sep='\\t', header=None,\n",
    "                         names=['text', 'label'], dtype={'text': str, 'label': str})\n",
    "train_data[\"label\"] = train_data[\"label\"].map(iddict)\n",
    "train_dataset = Dataset.from_pandas(train_data)\n",
    "\n",
    "val_data = pd.read_csv('dataset/NLP_ass_valid.tsv', sep='\\t', header=None,\n",
    "                       names=['text', 'label'], dtype={'text': str, 'label': str})\n",
    "val_data[\"label\"] = val_data[\"label\"].map(iddict)\n",
    "val_dataset = Dataset.from_pandas(val_data)\n",
    "\n",
    "test_data = pd.read_csv('dataset/NLP_ass_test.tsv',\n",
    "                        sep='\\t', header=None,\n",
    "                        names=['text', 'label'], dtype={'text': str, 'label': str})\n",
    "test_data[\"label\"] = test_data[\"label\"].map(iddict)\n",
    "test_dataset = Dataset.from_pandas(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 🌲Load Environment Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_NAME = os.getenv(\"MODEL_NAME\")\n",
    "REPOSITORY_NAME = os.getenv(\"REPOSITORY_NAME\")\n",
    "cache_dir = \"./cache/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 🎶 Fine-tune Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of T5ForSequenceClassification were not initialized from the model checkpoint at google/flan-t5-small and are newly initialized: ['classification_head.dense.bias', 'classification_head.dense.weight', 'classification_head.out_proj.bias', 'classification_head.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "config = AutoConfig.from_pretrained(\n",
    "    MODEL_NAME, num_labels=len(labeldict), iddict=iddict, labeldict=labeldict\n",
    ")\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    MODEL_NAME, config=config, cache_dir=cache_dir)\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    num_train_epochs=30,\n",
    "    output_dir=REPOSITORY_NAME,\n",
    "    logging_strategy=\"steps\",\n",
    "    logging_steps=100,\n",
    "    report_to=\"tensorboard\",\n",
    "    per_device_train_batch_size=8,\n",
    "    per_device_eval_batch_size=8,\n",
    "    fp16=False,  # Overflows with fp16\n",
    "    learning_rate=3e-4,\n",
    "    save_strategy=\"epoch\",\n",
    "    save_total_limit=2,\n",
    "    load_best_model_at_end=False,\n",
    "    push_to_hub=True,\n",
    "    hub_strategy=\"every_save\",\n",
    "    hub_model_id=REPOSITORY_NAME,\n",
    "    hub_token=HfFolder.get_token(),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 15383/15383 [00:06<00:00, 2303.78 examples/s]\n",
      "Map: 100%|██████████| 1922/1922 [00:00<00:00, 2281.36 examples/s]\n",
      "Map: 100%|██████████| 1924/1924 [00:00<00:00, 2297.99 examples/s]\n"
     ]
    }
   ],
   "source": [
    "def tkf(sample):\n",
    "    return tokenizer(sample[\"text\"], padding=\"max_length\", truncation=True)\n",
    "\n",
    "\n",
    "train_dataset_tokenized = train_dataset.map(tkf, batched=True)\n",
    "val_dataset_tokenized = val_dataset.map(tkf, batched=True)\n",
    "test_dataset_tokenized = test_dataset.map(tkf, batched=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 🏃🏻‍♂️‍➡️ Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def metric(prediction):\n",
    "    logits, labels = prediction\n",
    "    predictions = np.argmax(logits[0], axis=-1)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(\n",
    "        labels, predictions, average=\"binary\"\n",
    "    )\n",
    "    return {\"precision\": precision, \"recall\": recall, \"f1\": f1}\n",
    "\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "\n",
    "    train_dataset=train_dataset_tokenized,\n",
    "\n",
    "    eval_dataset=val_dataset_tokenized,\n",
    "    compute_metrics=metric,\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "\n",
    "\n",
    "tokenizer.save_pretrained(REPOSITORY_NAME)\n",
    "\n",
    "trainer.create_model_card()\n",
    "\n",
    "trainer.push_to_hub()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 🏗️ Load the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME)\n",
    "model.to(\"cuda\") if torch.cuda.is_available() else model.to(\"cpu\")\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 📊 Classify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify(texts):\n",
    "    inputs = tokenizer(\n",
    "        texts,\n",
    "        return_tensors=\"pt\",\n",
    "        max_length=512,\n",
    "        truncation=True,\n",
    "        padding=True,\n",
    "    )\n",
    "    inputs = inputs.to(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "    with torch.no_grad():\n",
    "        outputs = model(**inputs)\n",
    "\n",
    "    probs = torch.nn.functional.softmax(outputs.logits, dim=-1)\n",
    "\n",
    "    confidences, predicted_classes = torch.max(probs, dim=1)\n",
    "    predicted_classes = predicted_classes.cpu().numpy()\n",
    "    confidences = confidences.cpu().numpy()\n",
    "\n",
    "    predicted_labels = [labeldict[class_id] for class_id in predicted_classes]\n",
    "\n",
    "    return list(zip(predicted_labels, confidences))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 🏋🏻‍♂️ Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "  hatespeech       0.71      0.78      0.79       594\n",
      "      normal       0.75      0.71      0.77       782\n",
      "   offensive       0.56      0.46      0.52       548\n",
      "\n",
      "    accuracy                           0.72      1924\n",
      "   macro avg       0.68      0.68      0.70      1924\n",
      "weighted avg       0.72      0.71      0.72      1924\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def evalModel(batch_size=16):\n",
    "    pred = []\n",
    "    labels = []\n",
    "\n",
    "    for i in range(0, len(test_dataset), batch_size):\n",
    "        batch_predictions = classify(test_dataset[\"text\"][i : i + batch_size])\n",
    "\n",
    "        pred.extend(batch_predictions)\n",
    "        labels.extend([labeldict[label_id] for label_id in test_dataset[\"label\"][i : i + batch_size]])\n",
    "\n",
    "    report = classification_report(labels, [pair[0] for pair in pred])\n",
    "    \n",
    "    return report\n",
    "\n",
    "result = evalModel()\n",
    "\n",
    "print()\n",
    "print(\"Classification report:\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 🤼 Dataset Intersection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of common samples between train and test set:  5\n",
      "Intersection percentage:  0.0026001040041601664\n",
      "\n",
      "Number of common samples between validation and test set:  1\n",
      "Intersection percentage:  0.0005200208008320333\n"
     ]
    }
   ],
   "source": [
    "train_set = set(train_data['text'])\n",
    "val_set = set(val_data['text'])\n",
    "test_set = set(test_data['text'])\n",
    "\n",
    "def calculate_intersections(train_set, val_set, test_set):\n",
    "    return len(test_set.intersection(train_set)), len(test_set.intersection(val_set))\n",
    "\n",
    "train_test, validation_test = calculate_intersections(train_set, val_set, test_set)\n",
    "\n",
    "print(\"Number of common samples between train and test set: \", train_test)\n",
    "print(\"Intersection percentage: \", train_test / len(test_set))\n",
    "print()\n",
    "print(\"Number of common samples between validation and test set: \", validation_test)\n",
    "print(\"Intersection percentage: \", validation_test / len(test_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "Prasanna Paithankar (21CS30065)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}