gpt-and-bert_classification.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":8579914,"sourceType":"datasetVersion","datasetId":5131089}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-01T20:25:31.862523Z","iopub.execute_input":"2024-06-01T20:25:31.862965Z","iopub.status.idle":"2024-06-01T20:25:31.877550Z","shell.execute_reply.started":"2024-06-01T20:25:31.862935Z","shell.execute_reply":"2024-06-01T20:25:31.876291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom sklearn.model_selection import train_test_split\nfrom transformers import BertTokenizer, BertForSequenceClassification\nfrom transformers import GPT2Tokenizer, GPT2ForSequenceClassification\nfrom torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\nfrom transformers import AdamW, get_linear_schedule_with_warmup\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder\nfrom collections import Counter\n\nimport json","metadata":{"execution":{"iopub.status.busy":"2024-06-09T20:41:35.647762Z","iopub.execute_input":"2024-06-09T20:41:35.648188Z","iopub.status.idle":"2024-06-09T20:41:47.060526Z","shell.execute_reply.started":"2024-06-09T20:41:35.648155Z","shell.execute_reply":"2024-06-09T20:41:47.059017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_excel(\"/kaggle/input/cdc-2021-text-sequence-dataset/Text_Sequence_Dataset.xlsx\")","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:25:31.890109Z","iopub.execute_input":"2024-06-01T20:25:31.890549Z","iopub.status.idle":"2024-06-01T20:25:34.217120Z","shell.execute_reply.started":"2024-06-01T20:25:31.890516Z","shell.execute_reply":"2024-06-01T20:25:34.215918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:25:34.219605Z","iopub.execute_input":"2024-06-01T20:25:34.219972Z","iopub.status.idle":"2024-06-01T20:25:34.235885Z","shell.execute_reply.started":"2024-06-01T20:25:34.219939Z","shell.execute_reply":"2024-06-01T20:25:34.234597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_texts, test_texts, train_labels, test_labels = train_test_split(df['Sentence'], df['Classification'], test_size=0.20, random_state=42)\n\nlabel_encoder = LabelEncoder().fit(train_labels)\ntrain_labels = label_encoder.transform(train_labels)\ntest_labels = label_encoder.transform(test_labels)\n\nbert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\nbert_train_encodings = bert_tokenizer(train_texts.tolist(), truncation=True, padding=True, return_tensors='pt')\nbert_test_encodings = bert_tokenizer(test_texts.tolist(), truncation=True, padding=True, return_tensors='pt')\n\nbert_train_dataset = TensorDataset(bert_train_encodings['input_ids'], bert_train_encodings['attention_mask'], torch.tensor(train_labels))\nbert_test_dataset = TensorDataset(bert_test_encodings['input_ids'], bert_test_encodings['attention_mask'], torch.tensor(test_labels))\n\nbert_train_loader = DataLoader(bert_train_dataset, batch_size=8, shuffle=True)\nbert_test_loader = DataLoader(bert_test_dataset, batch_size=8, shuffle=False)\n\ngpt2_tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\ngpt2_tokenizer.pad_token = gpt2_tokenizer.eos_token\ngpt2_train_encodings = gpt2_tokenizer(train_texts.tolist(), truncation=True, padding=True, return_tensors='pt')\ngpt2_test_encodings = gpt2_tokenizer(test_texts.tolist(), truncation=True, padding=True, return_tensors='pt')\n\ngpt2_train_dataset = TensorDataset(gpt2_train_encodings['input_ids'], gpt2_train_encodings['attention_mask'], torch.tensor(train_labels))\ngpt2_test_dataset = TensorDataset(gpt2_test_encodings['input_ids'], gpt2_test_encodings['attention_mask'], torch.tensor(test_labels))\n\ngpt2_train_loader = DataLoader(gpt2_train_dataset, batch_size=8, shuffle=True)\ngpt2_test_loader = DataLoader(gpt2_test_dataset, batch_size=8, shuffle=False)\n\nsoftmax = torch.nn.Softmax(dim=-1) # a surprise tool that will help us later","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:25:34.237536Z","iopub.execute_input":"2024-06-01T20:25:34.237985Z","iopub.status.idle":"2024-06-01T20:27:37.502658Z","shell.execute_reply.started":"2024-06-01T20:25:34.237937Z","shell.execute_reply":"2024-06-01T20:27:37.501474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbert_model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nbert_model.to(device)\n\nclass_weights = torch.tensor([1.0, 1.0], dtype=torch.float)\n\noptimizer = torch.optim.AdamW(bert_model.parameters(), lr=1e-5)\ncriterion = torch.nn.CrossEntropyLoss(weight=class_weights)\n\nbert_model.train()\nfor epoch in range(4):  # Four epochs\n    for batch in bert_train_loader:\n        input_ids, attention_mask, labels = batch\n        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n        outputs = bert_model(input_ids, attention_mask=attention_mask, labels=labels)\n        loss = outputs.loss\n        loss.backward()\n        optimizer.step()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-01T20:27:37.504100Z","iopub.execute_input":"2024-06-01T20:27:37.504451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nbert_model.eval()\ncorrect = 0\ntotal = 0\n\npred = []\nprob = []\ntrue = []\n\nwith torch.no_grad():\n    for batch in bert_test_loader:\n        input_ids, attention_mask, labels = batch\n        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n\n        outputs = bert_model(input_ids, attention_mask=attention_mask)\n        _, predicted = torch.max(outputs.logits, 1)\n\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n        \n        pred = pred + predicted.tolist()\n        prob = prob + softmax(outputs.logits).tolist()\n        true = true + labels.tolist()\n\naccuracy = correct / total\nprint(f'Accuracy on test set: {accuracy}')\n\nwith open(\"bert_results.json\",\"w\") as f:\n    json.dump({\"predicted\":pred,\"probabilities\":prob,\"labels\":true},f)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ngpt2_model = GPT2ForSequenceClassification.from_pretrained('gpt2', num_labels=2)\ngpt2_model.config.pad_token_id = gpt2_model.config.eos_token_id\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ngpt2_model.to(device)\n\nclass_weights = torch.tensor([1.0, 1.0], dtype=torch.float)\n\noptimizer = torch.optim.AdamW(gpt2_model.parameters(), lr=1e-5)\ncriterion = torch.nn.CrossEntropyLoss(weight=class_weights)\n\ngpt2_model.train()\nfor epoch in range(4):\n    for batch in gpt2_train_loader:\n        input_ids, attention_mask, labels = batch\n        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n        outputs = gpt2_model(input_ids, attention_mask=attention_mask, labels=labels)\n        loss = outputs.loss\n        loss.backward()\n        optimizer.step()\n        ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ngpt2_model.eval()\ncorrect = 0\ntotal = 0\n\npred = []\nprob = []\ntrue = []\n\nwith torch.no_grad():\n    for batch in gpt2_test_loader:\n        input_ids, attention_mask, labels = batch\n        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n\n        outputs = gpt2_model(input_ids, attention_mask=attention_mask)\n        _, predicted = torch.max(outputs.logits, 1)\n        \n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n        \n        pred = pred + predicted.tolist()\n        prob = prob + softmax(outputs.logits).tolist()\n        true = true + labels.tolist()\n\naccuracy = correct / total\nprint(f'Accuracy on test set: {accuracy}')\n\nwith open(\"gpt2_results.json\",\"w\") as f:\n    json.dump({\"predicted\":pred,\"probabilities\":prob,\"labels\":true},f)\n","metadata":{},"execution_count":null,"outputs":[]}]}