gpt_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e5a1f180-e963-4397-85f2-f875698c5f91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully wrote the first 70000 lines to ./data/mr_70000.txt.\n"
     ]
    }
   ],
   "source": [
    "# Function to read the first k lines from the input file and write them to the output file\n",
    "def read_and_write_first_k_lines(input_file, output_file, num_lines=1000):\n",
    "    try:\n",
    "        with open(input_file, 'r') as infile, open(output_file, 'w') as outfile:\n",
    "            for i in range(num_lines):\n",
    "                line = infile.readline()\n",
    "                if not line:  # End of file reached before 1000 lines\n",
    "                    break\n",
    "                outfile.write(line)\n",
    "        print(f\"Successfully wrote the first {num_lines} lines to {output_file}.\")\n",
    "    except Exception as e:\n",
    "        print(f\"An error occurred: {e}\")\n",
    "\n",
    "# Call the function\n",
    "k = 70000\n",
    "input_file_path = './data/mr.txt'\n",
    "output_file_path = f\"./data/mr_{k}.txt\"\n",
    "device = 'cpu'\n",
    "if torch.cuda.is_available():\n",
    "    device = 'cuda'\n",
    "elif torch.backends.mps.is_available():\n",
    "    device = 'mps'\n",
    "\n",
    "read_and_write_first_k_lines(input_file_path, output_file_path, k)\n",
    "\n",
    "data_file = output_file_path\n",
    "with open(data_file, 'r') as file:\n",
    "    lines = file.readlines()\n",
    "\n",
    "\n",
    "# GET VOCAB\n",
    "vocab = set()\n",
    "sos_char = '♣'\n",
    "eos_char = '♦'\n",
    "for line in lines:\n",
    "    if line.strip() != \"\":\n",
    "        line = sos_char + line.strip() + eos_char\n",
    "        for ch in line:\n",
    "            vocab.add(ch)\n",
    "vocab = list(vocab)\n",
    "vocab_size = len(vocab)\n",
    "\n",
    "# CHARACTER ENCODER-DECODER\n",
    "s_to_i = {char: i for i, char in enumerate(vocab)}\n",
    "i_to_s = {i: char for i, char in enumerate(vocab)}\n",
    "encode = lambda x: [s_to_i[char] for char in x]\n",
    "decode = lambda x: \"\".join([i_to_s[num] for num in x])\n",
    "\n",
    "def generate_sentences(max_tokens):\n",
    "    feed = s_to_i[sos_char]\n",
    "    inp = torch.zeros(1, 1, dtype=torch.long, device=device)\n",
    "    inp[0][0] = feed\n",
    "    generarted_text = model.generate(inp, max_tokens)\n",
    "    print(decode(generarted_text.cpu().numpy()[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "9e327b3a-a416-43c8-b916-1e1d1dcb8cf8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/nx/5v1l58rx7b3b75sr_dn7_7mr0000gq/T/ipykernel_89731/954702899.py:5: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  model.load_state_dict(torch.load(weights_path, map_location=device))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BiagramLanguageModel(\n",
       "  (embedding_table): Embedding(360, 516)\n",
       "  (position_table): Embedding(64, 516)\n",
       "  (blocks): Sequential(\n",
       "    (0): Block(\n",
       "      (sa_heads): MultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-3): 4 x Head(\n",
       "            (key): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (query): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (value): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (projection): Linear(in_features=516, out_features=516, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "      (feed_forward): FeedForward(\n",
       "        (net): Sequential(\n",
       "          (0): Linear(in_features=516, out_features=2064, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2064, out_features=516, bias=True)\n",
       "          (3): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (norm1): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (1): Block(\n",
       "      (sa_heads): MultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-3): 4 x Head(\n",
       "            (key): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (query): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (value): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (projection): Linear(in_features=516, out_features=516, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "      (feed_forward): FeedForward(\n",
       "        (net): Sequential(\n",
       "          (0): Linear(in_features=516, out_features=2064, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2064, out_features=516, bias=True)\n",
       "          (3): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (norm1): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (2): Block(\n",
       "      (sa_heads): MultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-3): 4 x Head(\n",
       "            (key): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (query): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (value): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (projection): Linear(in_features=516, out_features=516, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "      (feed_forward): FeedForward(\n",
       "        (net): Sequential(\n",
       "          (0): Linear(in_features=516, out_features=2064, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2064, out_features=516, bias=True)\n",
       "          (3): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (norm1): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (3): Block(\n",
       "      (sa_heads): MultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-3): 4 x Head(\n",
       "            (key): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (query): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (value): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (projection): Linear(in_features=516, out_features=516, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "      (feed_forward): FeedForward(\n",
       "        (net): Sequential(\n",
       "          (0): Linear(in_features=516, out_features=2064, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2064, out_features=516, bias=True)\n",
       "          (3): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (norm1): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (4): Block(\n",
       "      (sa_heads): MultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-3): 4 x Head(\n",
       "            (key): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (query): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (value): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (projection): Linear(in_features=516, out_features=516, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "      (feed_forward): FeedForward(\n",
       "        (net): Sequential(\n",
       "          (0): Linear(in_features=516, out_features=2064, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2064, out_features=516, bias=True)\n",
       "          (3): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (norm1): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (5): Block(\n",
       "      (sa_heads): MultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-3): 4 x Head(\n",
       "            (key): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (query): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (value): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (projection): Linear(in_features=516, out_features=516, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "      (feed_forward): FeedForward(\n",
       "        (net): Sequential(\n",
       "          (0): Linear(in_features=516, out_features=2064, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2064, out_features=516, bias=True)\n",
       "          (3): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (norm1): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (6): Block(\n",
       "      (sa_heads): MultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-3): 4 x Head(\n",
       "            (key): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (query): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (value): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (projection): Linear(in_features=516, out_features=516, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "      (feed_forward): FeedForward(\n",
       "        (net): Sequential(\n",
       "          (0): Linear(in_features=516, out_features=2064, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2064, out_features=516, bias=True)\n",
       "          (3): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (norm1): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (7): Block(\n",
       "      (sa_heads): MultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-3): 4 x Head(\n",
       "            (key): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (query): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (value): Linear(in_features=516, out_features=129, bias=True)\n",
       "            (dropout): Dropout(p=0.2, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (projection): Linear(in_features=516, out_features=516, bias=True)\n",
       "        (dropout): Dropout(p=0.2, inplace=False)\n",
       "      )\n",
       "      (feed_forward): FeedForward(\n",
       "        (net): Sequential(\n",
       "          (0): Linear(in_features=516, out_features=2064, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2064, out_features=516, bias=True)\n",
       "          (3): Dropout(p=0.2, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (norm1): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "      (norm2): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (norm): LayerNorm((516,), eps=1e-05, elementwise_affine=True)\n",
       "  (lm_head): Linear(in_features=516, out_features=360, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gpt import BiagramLanguageModel\n",
    "\n",
    "weights_path = \"/Users/mayurb/src/open/marathiModels/weights/gpt2_marathi.pth\"\n",
    "model = BiagramLanguageModel()\n",
    "model.load_state_dict(torch.load(weights_path, map_location=device))\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "06340416-22de-4252-9b84-ba676f544124",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "♣P+ृъ♦झr॔♦ր0झъ]րड़झ⟶Pय)_उп)झսउPॉृ♦झञॐ⟶)॔♦॔♦झि०४್ड़Pझृउॉयँझ०)०)ր०िॐझह०սъ)P0―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――झ⟶ॉ子ր♦0लझP०)սPझ⟶P)य़नड़झञ)⟶Pउл☆ъं―0――――――――झP+Дउय―झञॐृGउп♦झ०४dॐրझ್+e)ऑ♦झrि)ъझK―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――ड़झबॐրPड़սъउп)०Pझ०)ս)बPउनिझ०♦न४⟶झ०Pउlझृ)್नृ)րउPझृGउп♦झД)ॐД―սØPँिगझ್+ञ♦०)झ⟶♦॔)ञ)№ड़झृ४PउД♦झ७)⟶ञॐъ♦झ⟶♦॔ड़झॉ子ր♦0――――――――――――――――――――――――――――――――――――――――――――――――――झe)िउृ)ृ)ъ)झञ子रॐGड़झ⟶)ऑ♦७झãञ)झ०उп⟶उl)झ್४०P―п)ि♦झञ)P)०Pिउि)e⟶झञP⟶)॔ड़――――――――――――――――――――――――――िउिउп७⟶झञ४॔)0―――――――――――――――――――――झrॉ♦0――――――――――――झп♦झृ)ಣ)झсP♦न子ृъ)_उп)ॐिड़झe)ДPझe)िPउृड़झ್)್॔॔उп)झृ)ॉ४ि)ॐъउп)झृड़झञ)०झսॉझãि♦ॉड़झृ子॔)|―'झД)ऑड़झ.॔Д№♦ъड़――――――――――――――――――――――――――――――――――――――――――――――――――――――――――♦झ್子िउп)ञ)№ड़गझ०րड़॔झիս⟶)Pր४ॐДञॐДङझrP子 ♦ս♦P॔)झreझि)ДP)P॔)झऑउп)ॐि)झैरड़७झп)ॐिड़झ್ॉ)झrսगझञPउДրझ⟶Pր)ि)ि♦झãि♦झॉ♦झrY―――――――――――――――――झ⟶ड़झ⟶)णДउP♦ъड़―झॉ)झе♦नड़――――ड़――――――――――――――――――ॐरॐGय)झreझeि)ञ)ॐऑझ⟶子րृड़िॐրPझॉ)ृG♦झãञ॔उп)झॊजझրड़झ⟶ड़झईऌझृ)ि)झP)॔)झॉ子Y―――″―झr್пझս)P⟶)Pउп⟶उ⟶♦ъउր)झãPउd)िउп)रրझ.ऑ子eिञ\n"
     ]
    }
   ],
   "source": [
    "generate_sentences(1000)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}