whisper_embeddings_faiss.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cecf5a8a",
   "metadata": {},
   "source": [
    "# **<h1 align =\"middle\"><b>Speech to Text, Generating Embeddings, FAISS</b></h1>**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37b29bc9",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3dbf5a2",
   "metadata": {},
   "source": [
    "> # **Whisper, Embeddings, FAISS**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e4999bf",
   "metadata": {},
   "source": [
    "> - 1. Speech to Text using Whisper\n",
    "> - (2a. Store / Query transcribed texts from ADW)\n",
    "> - 2b. Creating Embeddings\n",
    "> - 3. Apply FAISS for index-optimalisation\n",
    "> - 4. Save Index and Embeddings as Pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "896f42ef",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa6e94b0",
   "metadata": {},
   "source": [
    "## **Imports**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02f42b86",
   "metadata": {},
   "outputs": [],
   "source": [
    "#use the tensforlow conda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17f7c11e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!git clone https://github.com/kstathou/vector_engine\n",
    "#!pip install -r ./vector_engine/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c2921ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import ocifs\n",
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "import whisper\n",
    "import torchaudio\n",
    "from tqdm.notebook import tqdm\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import faiss\n",
    "import pickle\n",
    "import ffmpeg\n",
    "from vector_engine.vector_engine.utils import vector_search, id2details\n",
    "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "# !git clone https://huggingface.co/spaces/openai/whisper\n",
    "# %cd whisper\n",
    "# !pip install -r requirements.txt\n",
    "#!pip install jiwer\n",
    "# !pip install torchaudio\n",
    "#!pip install git+https://github.com/openai/whisper.git \n",
    "# #!pip install sentence_transformers\n",
    "# #!pip install faiss-cpu\n",
    "# !pip install ffmpeg\n",
    "# !pip install ffmpeg-python"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28625e11",
   "metadata": {},
   "source": [
    "# **1. Speech to Text using Whisper**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40dc4be9",
   "metadata": {},
   "source": [
    "## **1.1. Fetch recording from Bucket**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51f17a82",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for demo delete entire local folder\n",
    "!rm -r /home/datascience/input_recording"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19786959",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create local folder (in job)\n",
    "path_input_locally = \"/home/datascience/input_recording/\" \n",
    "\n",
    "try:       \n",
    "    if not os.path.exists(path_input_locally):         \n",
    "        os.makedirs(path_input_locally)    \n",
    "\n",
    "except OSError: \n",
    "    print ('Error: Creating directory of input recording')\n",
    "\n",
    "#copy recording from bucket to local folder\n",
    "fs = ocifs.OCIFileSystem()\n",
    "fs.invalidate_cache(\"oci://West_BP@frqap2zhtzbe/*.mp3\")\n",
    "fs.get(\"oci://West_BP@frqap2zhtzbe/*.mp3\", path_input_locally , recursive=True, refresh=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e6cc95e",
   "metadata": {},
   "source": [
    "## **1.2 Running Whisper**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bff7bea4",
   "metadata": {},
   "source": [
    "### **1.2.1 Detect Language (incorporated in model)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c120c2e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load whisper model\n",
    "model = whisper.load_model(\"medium\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bee72d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#for each recording in the folder detect langauge\n",
    "for recording in os.listdir(path_input_locally):\n",
    "        if (recording.endswith(\".mp3\")):\n",
    "            \n",
    "            audio_recording = os.path.join(path_input_locally, recording)\n",
    "            \n",
    "            # load audio and pad/trim it to fit 30 seconds\n",
    "            audio = whisper.load_audio(audio_recording)\n",
    "            audio = whisper.pad_or_trim(audio)\n",
    "            \n",
    "\n",
    "            # make log-Mel spectrogram and move to the same device as the model\n",
    "            mel = whisper.log_mel_spectrogram(audio).to(model.device)\n",
    "\n",
    "            # detect the spoken language\n",
    "            _, probs = model.detect_language(mel)\n",
    "            print(f\"Detected language: {max(probs, key=probs.get)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89a6639b",
   "metadata": {},
   "source": [
    "### **1.2.2 Load model and Run Transcription**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f622f6b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = []\n",
    "for recording in os.listdir(path_input_locally):\n",
    "        if (recording.endswith(\".mp3\")):\n",
    "            \n",
    "            audio_recording = os.path.join(path_input_locally, recording)\n",
    "            \n",
    "            #transcribe recording\n",
    "            result = model.transcribe(audio_recording)\n",
    "            \n",
    "            #append result for each recording to list\n",
    "            output.append(result['text'])\n",
    "                       \n",
    "            print(recording + \" is transcribed\")\n",
    "\n",
    "#all in dataframe\n",
    "df_transcriptions = pd.DataFrame(output, columns=['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b67d76ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#show\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "df_transcriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b891959",
   "metadata": {},
   "source": [
    "# **2. Creating Embeddings**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ea81885",
   "metadata": {},
   "outputs": [],
   "source": [
    "# see: https://towardsdatascience.com/how-to-build-a-semantic-search-engine-with-transformers-and-faiss-dcbea307a0e8\n",
    "\n",
    "# look into language-agnostic embeddings: https://ai.googleblog.com/2020/08/language-agnostic-bert-sentence.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc49f096",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create an id column. \n",
    "df_transcriptions['id_index'] = df_transcriptions.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af5f1c0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate the sentence-level DistilBERT\n",
    "model = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens')\n",
    "\n",
    "# Check if CUDA is available ans switch to GPU\n",
    "if torch.cuda.is_available():\n",
    "    model = model.to(torch.device(\"cuda\"))\n",
    "print(model.device)\n",
    "\n",
    "# Convert abstracts to vectors\n",
    "embeddings = model.encode(df_transcriptions.text.to_list(), show_progress_bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c157e5d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86dfea81",
   "metadata": {},
   "source": [
    "# **3. Apply FAISS**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47486acb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1: Change data type\n",
    "embeddings = np.array([embedding for embedding in embeddings]).astype(\"float32\")\n",
    "\n",
    "# Step 2: Instantiate the index\n",
    "index = faiss.IndexFlatL2(embeddings.shape[1])   #computes distances\n",
    "\n",
    "# Step 3: Pass the index to IndexIDMap\n",
    "index = faiss.IndexIDMap(index)\n",
    "\n",
    "# Step 4: Add vectors and their IDs\n",
    "index.add_with_ids(embeddings, df_transcriptions.id_index.values)\n",
    "\n",
    "print(f\"Number of vectors in the Faiss index: {index.ntotal}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b53788e4",
   "metadata": {},
   "source": [
    "## **3.1 Save embeddings and index as pickle**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb5623db",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('doc_embedding.pickle', 'wb') as pkl:\n",
    "    pickle.dump(embeddings, pkl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f67d341b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('doc_index.pickle', 'wb') as pkl:\n",
    "    pickle.dump(index, pkl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5be92111",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #open embeddings from pickle\n",
    "# with open('doc_embedding.pickle', 'rb') as pkl:\n",
    "#     doc_embedding = pickle.load(pkl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b72a59e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #open embeddings from pickle\n",
    "# with open('doc_index.pickle', 'rb') as pkl:\n",
    "#     doc_index = pickle.load(pkl)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27f54d26",
   "metadata": {},
   "source": [
    "## **3.2 Try an input example**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9395008",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set an example number and number of nearest neighbours\n",
    "x = 100\n",
    "kx = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e44fe66e",
   "metadata": {},
   "outputs": [],
   "source": [
    "## test\n",
    "user_query = \"I want to make a complaint about the marketing platform\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09bda6fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vector_search(query, model, index, num_results=10):\n",
    "    \"\"\"Tranforms query to vector using a pretrained, sentence-level\n",
    "    DistilBERT model and finds similar vectors using FAISS.\n",
    "    \n",
    "    Args:\n",
    "        query (str): User query that should be more than a sentence long.\n",
    "        model (sentence_transformers.SentenceTransformer.SentenceTransformer)\n",
    "        index (`numpy.ndarray`): FAISS index that needs to be deserialized.\n",
    "        num_results (int): Number of results to return.\n",
    "    \n",
    "    Returns:\n",
    "        D (:obj:`numpy.array` of `float`): Distance between results and query.\n",
    "        I (:obj:`numpy.array` of `int`): ID of the results.\n",
    "    \n",
    "    \"\"\"\n",
    "    vector = model.encode(list(query))\n",
    "    D, I = index.search(np.array(vector).astype(\"float32\"), k=num_results)\n",
    "    return D, I\n",
    "\n",
    "\n",
    "def id2details(df_transcriptions, I, column):\n",
    "    return [list(df_transcriptions[df_transcriptions.id_index == idx][column]) for idx in I[0]]\n",
    "\n",
    "# Querying the index\n",
    "D, I = vector_search([user_query], model, index, num_results=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "316f5e20",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_output = id2details(df_transcriptions, I, 'text')\n",
    "list_output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fb0cf3a",
   "metadata": {},
   "source": [
    "# **4. Create one file, one Job**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c83a9c8e",
   "metadata": {},
   "source": [
    "## **4.1 Create one .py File**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b860de11",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%writefile ./run_me_v1.py\n",
    "\n",
    "import os\n",
    "import ocifs\n",
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "import whisper\n",
    "import torchaudio\n",
    "from tqdm.notebook import tqdm\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import faiss\n",
    "import pickle\n",
    "import ffmpeg\n",
    "from vector_engine.vector_engine.utils import vector_search, id2details                           ## this is different for Job vs notebook because of folder structure\n",
    "DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "#fetch environmment variables:\n",
    "recording_name = os.environ.get(\"recording_name\", \"recording\")\n",
    "print(\"Fetching environment variable called \" + recording_name)\n",
    "\n",
    "#ffmpeg\n",
    "os.system(\"cp ./ffmpeg ./usr/bin/ffmpeg\")  #copy file to other direcotry\n",
    "os.system(\"cp ./ffprobe ./usr/bin/ffprobe\")\n",
    "\n",
    "os.system(\"chmod +rwx ./usr/bin/ffprobe  #change permission\")\n",
    "os.system(\"chmod +rwx ./usr/bin/ffmpeg\")\n",
    "\n",
    "\n",
    "######## ----------------------------------------------------- #################\n",
    "# Step 1\n",
    "######## ----------------------------------------------------- #################\n",
    "\n",
    "#create local folder (in job)\n",
    "path_input_locally = \"/home/datascience/input_recording/\" \n",
    "\n",
    "try:       \n",
    "    if not os.path.exists(path_input_locally):         \n",
    "        os.makedirs(path_input_locally)    \n",
    "\n",
    "except OSError: \n",
    "    print ('Error: Creating directory of input recording')\n",
    "\n",
    "#copy recording from bucket to local folder\n",
    "fs = ocifs.OCIFileSystem()\n",
    "fs.invalidate_cache(\"oci://West_BP@frqap2zhtzbe/*.mp3\")\n",
    "fs.get(\"oci://West_BP@frqap2zhtzbe/*.mp3\", path_input_locally , recursive=True, refresh=True)\n",
    "\n",
    "######## ----------------------------------------------------- #################\n",
    "# Step 2 - Load model and Detect languages\n",
    "######## ----------------------------------------------------- #################\n",
    "\n",
    "#load whisper model\n",
    "model = whisper.load_model(\"base\")\n",
    "\n",
    "#for each recording in the folder detect langauge\n",
    "for recording in os.listdir(path_input_locally):\n",
    "        if (recording.endswith(\".mp3\")):\n",
    "            \n",
    "            audio_recording = os.path.join(path_input_locally, recording)\n",
    "            \n",
    "            # load audio and pad/trim it to fit 30 seconds\n",
    "            audio = whisper.load_audio(audio_recording)\n",
    "            audio = whisper.pad_or_trim(audio)\n",
    "            \n",
    "\n",
    "            # make log-Mel spectrogram and move to the same device as the model\n",
    "            mel = whisper.log_mel_spectrogram(audio).to(model.device)\n",
    "\n",
    "            # detect the spoken language\n",
    "            _, probs = model.detect_language(mel)\n",
    "            print(f\"Detected language: {max(probs, key=probs.get)}\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "######## ----------------------------------------------------- #################\n",
    "# Step 3 Transcripte recordings\n",
    "######## ----------------------------------------------------- #################\n",
    "\n",
    "output = []\n",
    "for recording in os.listdir(path_input_locally):\n",
    "        if (recording.endswith(\".mp3\")):\n",
    "            \n",
    "            audio_recording = os.path.join(path_input_locally, recording)\n",
    "            \n",
    "            #transcribe recording\n",
    "            result = model.transcribe(audio_recording)\n",
    "            \n",
    "            #append result for each recording to list\n",
    "            output.append(result['text'])\n",
    "                       \n",
    "            print(recording + \" is transcribed\")\n",
    "\n",
    "#all in dataframe\n",
    "df_transcriptions = pd.DataFrame(output, columns=['text'])\n",
    "\n",
    "######## ----------------------------------------------------- #################\n",
    "# Step 3 Create embeddings\n",
    "######## ----------------------------------------------------- #################\n",
    "\n",
    "#create an id column. \n",
    "df_transcriptions['id_index'] = df_transcriptions.index\n",
    "\n",
    "# Instantiate the sentence-level DistilBERT\n",
    "model = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens')\n",
    "\n",
    "# Check if CUDA is available ans switch to GPU\n",
    "if torch.cuda.is_available():\n",
    "    model = model.to(torch.device(\"cuda\"))\n",
    "print(model.device)\n",
    "\n",
    "# Convert abstracts to vectors\n",
    "embeddings = model.encode(df_transcriptions.text.to_list(), show_progress_bar=True)\n",
    "\n",
    "######## ----------------------------------------------------- #################\n",
    "# Step 4 Apply FAISS\n",
    "######## ----------------------------------------------------- #################\n",
    "\n",
    "# Step 1: Change data type\n",
    "embeddings = np.array([embedding for embedding in embeddings]).astype(\"float32\")\n",
    "\n",
    "# Step 2: Instantiate the index\n",
    "index = faiss.IndexFlatL2(embeddings.shape[1])\n",
    "\n",
    "# Step 3: Pass the index to IndexIDMap\n",
    "index = faiss.IndexIDMap(index)\n",
    "\n",
    "# Step 4: Add vectors and their IDs\n",
    "index.add_with_ids(embeddings, df_transcriptions.id_index.values)\n",
    "\n",
    "print(f\"Number of vectors in the Faiss index: {index.ntotal}\")\n",
    "\n",
    "print(\"-------------------------------------------------------\")\n",
    "print(\"-------------------------------------------------------\")\n",
    "print(\"Embeddings are created and FAISS is applied. Pickle files or embeddings can be pushed to Vector DB or anything else\")\n",
    "print(\"-------------------------------------------------------\")\n",
    "print(\"-------------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19ddcd82",
   "metadata": {},
   "source": [
    "## **4.2 Create and Trigger Job**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b93a9957",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # publish conda, as we added new/custom packages\n",
    "# !odsc conda init -b conda_environment_yolov5 -n frqap2zhtzbe -a resource_principal\n",
    "# !odsc conda publish -s tensorflow28_p38_gpu_v1 --force"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec2e3677",
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports\n",
    "from ads.common.oci_logging import OCILogGroup, OCILog\n",
    "from ads.jobs import Job, DataScienceJob, PythonRuntime\n",
    "from datetime import datetime, timedelta\n",
    "from ads import set_auth\n",
    "\n",
    "#authentication\n",
    "set_auth(auth='resource_principal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49411428",
   "metadata": {},
   "outputs": [],
   "source": [
    "## create the job\n",
    "\n",
    "job = (\n",
    "    Job(name=\"job_v1\")\n",
    "    .with_infrastructure(\n",
    "        DataScienceJob()\n",
    "        # Configure logging for getting the job run outputs.\n",
    "        .with_log_group_id(\"ocid1.loggroup.oc1.eu-frankfurt-1.amaaaaaangencdyajxalcuggjaug57r3ugare7olsk44ts2shyv7azqbxf4q\")\n",
    "        .with_shape_name(\"VM.Standard2.4\")\n",
    "        #.with_shape_config_details(memory_in_gbs=16, ocpus=5)\n",
    "        .with_block_storage_size(200)\n",
    "    )\n",
    "    .with_runtime(\n",
    "        PythonRuntime()\n",
    "        # Specify the service conda environment by slug name.\n",
    "        .with_custom_conda(\"oci://conda_environment_yolov5@frqap2zhtzbe/conda_environments/gpu/TensorFlow 2.8 for GPU on Python 3.8/1.0/tensorflow28_p38_gpu_v1\")\n",
    "        # Source code of the job, can be local or remote.\n",
    "        .with_source(\"/home/datascience/\")\n",
    "        #Environment variable\n",
    "        .with_environment_variable(recording_name=\"Default variable\")\n",
    "        .with_entrypoint(\"./run_me_v1.py\")\n",
    "    )\n",
    ")\n",
    "\n",
    "job.create()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7bb6358",
   "metadata": {},
   "outputs": [],
   "source": [
    "job_run_env = job.run(\n",
    "    name=\"job_run_v1\",\n",
    "    env_var={\"recording_name\": \"An example environment variable. Could passed to this Job\"}\n",
    ")\n",
    "\n",
    "job_run_watch = job_run_env.watch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9439342",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d3cd6ee",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow28_p38_gpu_v1]",
   "language": "python",
   "name": "conda-env-tensorflow28_p38_gpu_v1-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}