6_1_replacing_phoneme_pt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a158536-6e39-4582-9a14-c47c6b82df18",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "import torch\n",
    "import requests\n",
    "import json\n",
    "import os \n",
    "\n",
    "model_name=\"large-v2\"\n",
    "pattern = \"phoneme\"\n",
    "\n",
    "\n",
    "\n",
    "# In[2]:\n",
    "\n",
    "\n",
    "from dataclasses import dataclass, field, replace\n",
    "from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple, Union\n",
    "import whisper_openAI.whisper as whisper\n",
    "import torch\n",
    "from whisper_openAI.whisper.tokenizer import Tokenizer, get_tokenizer\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch import Tensor\n",
    "import json\n",
    "\n",
    "# We get the acoustic embeddings from Whisper Large V2\n",
    "model,processor = whisper.load_model(\"large-v2\")\n",
    "# model,processor = whisper.load_model(\"medium\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "12db7520-ae60-464d-aaca-9d2f957d713f",
   "metadata": {},
   "outputs": [],
   "source": [
    "speaker_id = 'M03'\n",
    "test_speaker = speaker_id\n",
    "pattern = \"phoneme\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "717cea24-a650-4566-be6b-789b53b559bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train has 9,376 samples\n"
     ]
    }
   ],
   "source": [
    "with open(f'Inference/gs_inferences/large-v2_hypo/torgo_train_{speaker_id}_{model_name}.json', \"r\") as file:  # Change the file path and name here\n",
    "    train_data = json.load(file)\n",
    "\n",
    "with open(f'Inference/gs_inferences/large-v2_hypo/torgo_val_{speaker_id}_{model_name}.json', \"r\") as valid_file:\n",
    "    val_data = json.load(valid_file)\n",
    "\n",
    "# Load the test set\n",
    "with open(f'Inference/gs_inferences/large-v2_hypo/torgo_test_{speaker_id}_{model_name}.json', \"r\") as test_file:\n",
    "    test_data = json.load(test_file)\n",
    "    \n",
    "\n",
    "from lit_llama.tokenizer import Tokenizer\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "tokenizer_path: Path = Path(\"weights/tokenizer.model\")\n",
    "tokenizer = Tokenizer(tokenizer_path)\n",
    "print(f\"train has {len(train_data):,} samples\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dec66233-7d1b-47b4-ba65-a64338d11d54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of old_data_train: 9376\n",
      "Length of old_data_val: 460\n",
      "Length of old_data_test: 442\n"
     ]
    }
   ],
   "source": [
    "    import torch\n",
    "    old_folder = \"tiny_hypo\"\n",
    "    old_data_train = torch.load(f'Inference/gs_inferences/{old_folder}/torgo_{test_speaker}_train.pt',map_location=torch.device('cpu'))\n",
    "    old_data_val = torch.load(f'Inference/gs_inferences/{old_folder}/torgo_{test_speaker}_val.pt',map_location=torch.device('cpu'))\n",
    "    old_data_test = torch.load(f'Inference/gs_inferences/{old_folder}/torgo_{test_speaker}_test.pt',map_location=torch.device('cpu'))\n",
    "    \n",
    "    \n",
    "    # In[8]:\n",
    "    \n",
    "    \n",
    "    # Print the lengths of the loaded data\n",
    "    print(f'Length of old_data_train: {len(old_data_train)}')\n",
    "    print(f'Length of old_data_val: {len(old_data_val)}')\n",
    "    print(f'Length of old_data_test: {len(old_data_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "92e48a18-36b6-48ca-938c-5a2309473c98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'F01-Session1-arrayMic-0009': {'temp': 0.74,\n",
       "  'path': '/work/van-speech-nlp/data/torgo/F01/Session1/wav_arrayMic/0009.wav',\n",
       "  'ground_truth': 'pat',\n",
       "  'inference': ['',\n",
       "   'ah',\n",
       "   'ah!',\n",
       "   'alright',\n",
       "   'bartender 3',\n",
       "   'berk',\n",
       "   'bleep',\n",
       "   'blurp',\n",
       "   'burp',\n",
       "   'burp!',\n",
       "   'burps',\n",
       "   'but',\n",
       "   'bye!',\n",
       "   'ehh',\n",
       "   'ha',\n",
       "   'hah',\n",
       "   'hehe',\n",
       "   'hot!',\n",
       "   'hrrrgh',\n",
       "   'huh',\n",
       "   'hurt!',\n",
       "   'huuuuh!',\n",
       "   'im gonna go',\n",
       "   'oh',\n",
       "   'purr!',\n",
       "   'rip camera man',\n",
       "   'thanks for watching see you next time!',\n",
       "   'ugh',\n",
       "   'uh',\n",
       "   'uhoh',\n",
       "   'urgh!'],\n",
       "  'source': 'NP-Torgo',\n",
       "  'category': 'NP-Torgo',\n",
       "  'time': 30.0}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "eb88066c-fb13-4eda-b757-1b62b6f5064b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(tokenizer: Tokenizer, string: str, max_length: int, eos=True) -> torch.Tensor:\n",
    "    return tokenizer.encode(string, bos=True, eos=eos, max_length=max_length)\n",
    "    \n",
    "def process_train_data(train_data, old_data):\n",
    "    # instruction = 'You are an ASR transcript selector. You have a few transcripts generated by an automatic speech recognition model. Your task is to generate the most likely transcript from them. If the generated transcripts have grammatical or logical errors, you will modify them accordingly to produce the most accurate and coherent transcript.'\n",
    "    instruction = 'You are an ASR transcript selector. The first line is the phoneme recognition result. The rest of the lines are a few transcripts generated by an automatic speech recognition model. Your task is to generate the most likely transcript from them. If the generated transcripts have grammatical or logical errors, you will modify them accordingly to produce the most accurate and coherent transcript.'\n",
    "    result = []\n",
    "\n",
    "    for i in tqdm(range(len(train_data))):        \n",
    "        for name in train_data[i].keys():\n",
    "            ip = train_data[i][name]\n",
    "        # print(ip)\n",
    "        inference = ip['inference']\n",
    "        gt = ip['ground_truth']\n",
    "        path = ip['path']\n",
    "\n",
    "        # Find the corresponding phoneme in the dataframe\n",
    "        matching_entry = df[df['audio'] == path]\n",
    "\n",
    "        phoneme = matching_entry[\"phoneme\"].values[0]\n",
    "        \n",
    "        # Removing the ground_truth, if present among the inferences for the prompt\n",
    "        if gt in inference:\n",
    "            inference.remove(gt)\n",
    "                \n",
    "        # Joining the inputs with '\\n'\n",
    "        for_input = f\"Phoneme Results: {phoneme} \\n\" + \"Possible transcripts: \\n\" + '\\n'.join(inference[:15])  # Add phoneme to the beginning of the input\n",
    "\n",
    "        print(path)\n",
    "        print(f\"{for_input} \\n\")\n",
    "        # The prompt follows the Alpaca template\n",
    "        full_prompt = f\"\"\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n{instruction}\\n\\n### Input:\\n{for_input}\\n\\n### Response:\"\"\"\n",
    "        full_prompt_and_response = full_prompt + gt\n",
    "\n",
    "        encoded_full_prompt = tokenize(tokenizer, full_prompt, max_length=2048, eos=False)\n",
    "        encoded_full_prompt_and_response = tokenize(tokenizer, full_prompt_and_response, eos=True, max_length=2048)\n",
    "        labels = encoded_full_prompt_and_response.clone()\n",
    "        labels_with_masked_input = encoded_full_prompt_and_response.clone()\n",
    "        labels_with_masked_input[:len(encoded_full_prompt)] = -1\n",
    "        \n",
    "        audio_features = old_data[i][\"audio_features\"]\n",
    "        \n",
    "        result.append({**ip, 'index': name, \"input_ids\": encoded_full_prompt_and_response, \"input_ids_no_response\": encoded_full_prompt, \"labels\": labels, 'labels_with_masked_input': labels_with_masked_input, 'audio_features': audio_features.bfloat16()})\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "c2d210bb-f62e-4338-9c96-fe50113cef78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session</th>\n",
       "      <th>audio</th>\n",
       "      <th>text</th>\n",
       "      <th>speaker_id</th>\n",
       "      <th>test_data</th>\n",
       "      <th>phoneme</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>F01-Session1-arrayMic-0006</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/F01/Session1/w...</td>\n",
       "      <td>STICK</td>\n",
       "      <td>F01</td>\n",
       "      <td>1</td>\n",
       "      <td>d e r t</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>F01-Session1-arrayMic-0008</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/F01/Session1/w...</td>\n",
       "      <td>EXCEPT IN THE WINTER WHEN THE OOZE OR SNOW OR ...</td>\n",
       "      <td>F01</td>\n",
       "      <td>1</td>\n",
       "      <td>a s ɛ ɪ n n ə i l i uː r ɑ s uː ɑ a h i v ɛ n k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>F01-Session1-arrayMic-0009</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/F01/Session1/w...</td>\n",
       "      <td>PAT</td>\n",
       "      <td>F01</td>\n",
       "      <td>0</td>\n",
       "      <td>p aː</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F01-Session1-arrayMic-0010</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/F01/Session1/w...</td>\n",
       "      <td>UP</td>\n",
       "      <td>F01</td>\n",
       "      <td>1</td>\n",
       "      <td>aː b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>F01-Session1-arrayMic-0011</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/F01/Session1/w...</td>\n",
       "      <td>MEAT</td>\n",
       "      <td>F01</td>\n",
       "      <td>0</td>\n",
       "      <td>m eɪ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16389</th>\n",
       "      <td>MC04-Session2-arrayMic-1022</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/MC04/Session2/...</td>\n",
       "      <td>WART</td>\n",
       "      <td>MC04</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16390</th>\n",
       "      <td>MC04-Session2-arrayMic-1023</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/MC04/Session2/...</td>\n",
       "      <td>THE FAMILY REQUESTS THAT FLOWERS BE OMITTED</td>\n",
       "      <td>MC04</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16391</th>\n",
       "      <td>MC04-Session2-arrayMic-1024</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/MC04/Session2/...</td>\n",
       "      <td>SHIP</td>\n",
       "      <td>MC04</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16392</th>\n",
       "      <td>MC04-Session2-arrayMic-1026</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/MC04/Session2/...</td>\n",
       "      <td>WHY</td>\n",
       "      <td>MC04</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16393</th>\n",
       "      <td>MC04-Session2-arrayMic-1027</td>\n",
       "      <td>/work/van-speech-nlp/data/torgo/MC04/Session2/...</td>\n",
       "      <td>TALL</td>\n",
       "      <td>MC04</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16394 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           session  \\\n",
       "0       F01-Session1-arrayMic-0006   \n",
       "1       F01-Session1-arrayMic-0008   \n",
       "2       F01-Session1-arrayMic-0009   \n",
       "3       F01-Session1-arrayMic-0010   \n",
       "4       F01-Session1-arrayMic-0011   \n",
       "...                            ...   \n",
       "16389  MC04-Session2-arrayMic-1022   \n",
       "16390  MC04-Session2-arrayMic-1023   \n",
       "16391  MC04-Session2-arrayMic-1024   \n",
       "16392  MC04-Session2-arrayMic-1026   \n",
       "16393  MC04-Session2-arrayMic-1027   \n",
       "\n",
       "                                                   audio  \\\n",
       "0      /work/van-speech-nlp/data/torgo/F01/Session1/w...   \n",
       "1      /work/van-speech-nlp/data/torgo/F01/Session1/w...   \n",
       "2      /work/van-speech-nlp/data/torgo/F01/Session1/w...   \n",
       "3      /work/van-speech-nlp/data/torgo/F01/Session1/w...   \n",
       "4      /work/van-speech-nlp/data/torgo/F01/Session1/w...   \n",
       "...                                                  ...   \n",
       "16389  /work/van-speech-nlp/data/torgo/MC04/Session2/...   \n",
       "16390  /work/van-speech-nlp/data/torgo/MC04/Session2/...   \n",
       "16391  /work/van-speech-nlp/data/torgo/MC04/Session2/...   \n",
       "16392  /work/van-speech-nlp/data/torgo/MC04/Session2/...   \n",
       "16393  /work/van-speech-nlp/data/torgo/MC04/Session2/...   \n",
       "\n",
       "                                                    text speaker_id  \\\n",
       "0                                                  STICK        F01   \n",
       "1      EXCEPT IN THE WINTER WHEN THE OOZE OR SNOW OR ...        F01   \n",
       "2                                                    PAT        F01   \n",
       "3                                                     UP        F01   \n",
       "4                                                   MEAT        F01   \n",
       "...                                                  ...        ...   \n",
       "16389                                               WART       MC04   \n",
       "16390        THE FAMILY REQUESTS THAT FLOWERS BE OMITTED       MC04   \n",
       "16391                                               SHIP       MC04   \n",
       "16392                                                WHY       MC04   \n",
       "16393                                               TALL       MC04   \n",
       "\n",
       "       test_data                                          phoneme  \n",
       "0              1                                          d e r t  \n",
       "1              1  a s ɛ ɪ n n ə i l i uː r ɑ s uː ɑ a h i v ɛ n k  \n",
       "2              0                                             p aː  \n",
       "3              1                                             aː b  \n",
       "4              0                                             m eɪ  \n",
       "...          ...                                              ...  \n",
       "16389          0                                              NaN  \n",
       "16390          0                                              NaN  \n",
       "16391          0                                              NaN  \n",
       "16392          0                                              NaN  \n",
       "16393          0                                              NaN  \n",
       "\n",
       "[16394 rows x 6 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv('data_preparation/torgo_phoneme.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "aeffe1a1-08e3-4416-86ed-ed9d1151fa3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4/4 [00:00<00:00, 159.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/work/van-speech-nlp/data/torgo/F01/Session1/wav_arrayMic/0009.wav\n",
      "Phoneme Results: p aː \n",
      "Possible transcripts: \n",
      "\n",
      "ah\n",
      "ah!\n",
      "alright\n",
      "bartender 3\n",
      "berk\n",
      "bleep\n",
      "blurp\n",
      "burp\n",
      "burp!\n",
      "burps\n",
      "but\n",
      "bye!\n",
      "ehh\n",
      "ha \n",
      "\n",
      "/work/van-speech-nlp/data/torgo/F01/Session1/wav_arrayMic/0012.wav\n",
      "Phoneme Results: i \n",
      "Possible transcripts: \n",
      "eat\n",
      "great\n",
      "heh heeh\n",
      "hey\n",
      "hey hey\n",
      "lets not get ahead of ourselves\n",
      "little asleep\n",
      "neat\n",
      "right\n",
      "wait\n",
      "wait wait\n",
      "we\n",
      "ねえ。 nee \n",
      "\n",
      "/work/van-speech-nlp/data/torgo/F01/Session1/wav_arrayMic/0015.wav\n",
      "Phoneme Results: eː \n",
      "Possible transcripts: \n",
      "a\n",
      "a!\n",
      "and\n",
      "aye\n",
      "aye!\n",
      "ayeee\n",
      "ayyy\n",
      "ayyyy!\n",
      "ayyyyyy\n",
      "bye\n",
      "eeeh\n",
      "eh\n",
      "ehh\n",
      "ehhh\n",
      "ehhhhh \n",
      "\n",
      "/work/van-speech-nlp/data/torgo/F01/Session1/wav_arrayMic/0016.wav\n",
      "Phoneme Results: f aː n \n",
      "Possible transcripts: \n",
      "food\n",
      "food!\n",
      "i\n",
      "sword\n",
      "the world\n",
      "wha\n",
      "what\n",
      "what!\n",
      "wood\n",
      "woop!\n",
      "wut \n",
      "\n",
      "Processed train data and saved checkpoint for M03\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "split = \"train\"\n",
    "result = process_train_data(, old_data_train)\n",
    "torch.save(result,f'Inference/gs_inferences/torgo_{speaker_id}_{model_name}_{pattern}_{split}.pt')\n",
    "print(f\"Processed {split} data and saved checkpoint for {speaker_id}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "963a58de-c0d0-46c1-8e06-06cd9cfec717",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9dba8c76-4d4e-4abd-a187-27e3f4e84f9b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e52306f6-087c-407c-8ef3-9a7a1a9e8975",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35a9f4f6-a6ca-4ab0-9df3-273a0499c55b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}