ContributionAnalysis_De_NB11042023.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <font color=\"00ff00\">  **Statistical Analysis for Accented Speech recognition**</font>\n",
    "<font color=white> **evaluates the performance of an ASR model regarding accented speech**</font> \n",
    "\n",
    "## <font color=\"00ff00\">  **Section 3.2, Information Mixing Analysis**</font>\n",
    "<font color=white> **calculate the gradient contributions**</font> \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pickle\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from itertools import chain \n",
    "from itertools import groupby\n",
    "import glob\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/mmm2050/QU_DFKI_Thesis/Experimentation/ASR_Accent_Analysis_De'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#####################################################################################\n",
    "########### set the Data files paths on Conda Notebook on Ubuntu #################### \n",
    "\n",
    "### \n",
    "os.chdir('/home/mmm2050/QU_DFKI_Thesis/Experimentation/ASR_Accent_Analysis_De')\n",
    "main_path=os.getcwd()\n",
    "\n",
    "if not os.path.exists(main_path+'/Data_results'):\n",
    "  os.makedirs(main_path+'/Data_results')\n",
    "\n",
    "if not os.path.exists(main_path+'/Figures_results'):\n",
    "  os.makedirs(main_path+'/Figures_results')\n",
    "\n",
    "Data_path=main_path+'/Data/'\n",
    "Data_results_path=main_path+'/Data_results/'\n",
    "json_file_path=Data_path+'results.json'\n",
    "validated_tsv_path=Data_path+'validated.tsv'\n",
    "validated_tsv_path_small=Data_path+'validated_small.tsv'\n",
    " \n",
    "\n",
    "validated_en_tsv_path=main_path+'/DeepSpeech/data/validated.tsv'\n",
    "validated_en_tsv_path_small=main_path+'/DeepSpeech/data/validated_small.tsv'\n",
    "# json_en_file_path=main_path+'/DeepSpeech/data/Results_En.json'\n",
    "#####################################################################################\n",
    "########### set the Data files paths on Colab Notebook###############################\n",
    "# # Import the dataset file by method1 \n",
    "# # from google.colab import files\n",
    "# # uploaded = files.upload()\n",
    "# if not os.path.exists(Data_path'):\n",
    "#   os.makedirs(Data_path')\n",
    "# ! cp /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/cv-corpus04072022/de/validated.tsv /content/Data/\n",
    "# # copy the expermintations files to deal with them\n",
    "# ! cp /content/drive/MyDrive/QU-DFKI-Thesis-ASR/Experimentation/ASR-Accent-Analysis-De/Data/*.* /content/Data/\n",
    "# # copy the expermintations files from Mozilla Commen Voice v 10 to deal with them\n",
    "\n",
    "# import shutil\n",
    "# shutil.rmtree('/content/audio', ignore_errors=True)\n",
    "# contribution_path = 'Contribution/'\n",
    "contribution_path = main_path+'/DeepSpeech/data/Contribution/timit_blank/'\n",
    "layers = ['conv','rnn_0','rnn_1','rnn_2','rnn_3','rnn_4']\n",
    "\n",
    " \n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setup "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=white> **Creat a EN dictionary for the Audio files , Accents** </font> \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('my_data/final-file-info.json', 'r') as j:\n",
    "# \tfile_meta = json.load(j)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Creat a EN dictionary for the Audio files , Accents#######\n",
    "###################################################################\n",
    "\n",
    "# initialize an empty dictionary\n",
    "file_meta = {}\n",
    "\n",
    "# read the TSV file\n",
    "with open(validated_en_tsv_path_small, 'r') as f:\n",
    "    # skip the header row\n",
    "    next(f)\n",
    "    # iterate over the remaining rows\n",
    "    for line in f:\n",
    "        # split the line into columns\n",
    "        cols = line.strip().split('\\t')\n",
    "        # extract the relevant columns\n",
    "        filename = cols[1].split('.')[0]\n",
    "        accent = cols[7]\n",
    "        transcript = cols[2]\n",
    "        # create a dictionary for this file\n",
    "        file_dict = {'accent': accent, 'transcript': transcript}\n",
    "        # add the dictionary to the file_meta dictionary\n",
    "        file_meta[filename] = file_dict\n",
    "        \n",
    "\n",
    "        # Open a new CSV file in write mode\n",
    "with open(main_path+'/DeepSpeech/data/file_meta.csv', 'w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "\n",
    "    # Write the header row\n",
    "    writer.writerow(file_meta.keys())\n",
    "\n",
    "    # Write the values row\n",
    "    writer.writerow(file_meta.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'common_voice_en_533247': {'accent': 'australia',\n",
       "  'transcript': 'The burning fire had been extinguished.'},\n",
       " 'common_voice_en_195627': {'accent': 'canada',\n",
       "  'transcript': 'The burning fire had been extinguished.'},\n",
       " 'common_voice_en_19574': {'accent': 'african',\n",
       "  'transcript': 'The burning fire had been extinguished.'}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title  Process \"align.json\" files contents and import end_times,phones to pass it to ###\n",
    "################################## file_meta dict ########################################\n",
    "##########################################################################################\n",
    "import json\n",
    "import csv\n",
    "import os\n",
    "\n",
    "'''\n",
    "##########################################################################################\n",
    "##### Input : the output align.json of an Audio (From Gentle Output)\n",
    "##### Output:\n",
    "##########################################################################################\n",
    "'''\n",
    "\n",
    "\n",
    "directory_path = main_path+'/DeepSpeech/data/gentle_results/' # replace with the actual path of your directory\n",
    "\n",
    "\n",
    "i=0\n",
    "\n",
    "file=main_path+'/DeepSpeech/data/gentle_results/aaaaa'\n",
    "dir_path = os.path.dirname(os.path.realpath(file))\n",
    "\n",
    "# Loop through the subdirectories and get the file names\n",
    "for subdirectory in os.listdir(directory_path):\n",
    "    subdirectory_path = os.path.join(directory_path, subdirectory)\n",
    "    # print(subdirectory)\n",
    "    # if os.path.isdir(subdirectory_path):\n",
    "        # print(f\"Files in {subdirectory}:\")\n",
    "        # for filename in os.listdir(subdirectory_path):\n",
    "            # print(filename)\n",
    "    Audio_File_ID=subdirectory      #exp. 'common_voice_en_19574'\n",
    "    file_path = dir_path + \"/\"+Audio_File_ID+\"/\"\n",
    "    csv_input = [[], []]\n",
    "    # csv_input = []\n",
    "\n",
    "    with open(file_path + \"align.json\", 'r') as file:\n",
    "        data = json.load(file)\n",
    "        words_array = data[\"words\"]\n",
    "        for i in range (len(words_array)):\n",
    "            phones_array = words_array[i][\"phones\"]\n",
    "            start_time = float(words_array[i][\"start\"])\n",
    "            end_time = start_time\n",
    "            for j in range (len(phones_array)):\n",
    "                end_time = round(end_time + float(words_array[i][\"phones\"][j][\"duration\"]), 2)\n",
    "                csv_input[0].append(words_array[i][\"phones\"][j][\"phone\"])\n",
    "                csv_input[1].append(end_time)\n",
    "\n",
    "                # phones = [p[6:] for p in sample['phones']]\n",
    "                \n",
    "                # update the file_meta dictionary with the phone transcriptions and their corresponding end times\n",
    "                file_meta[Audio_File_ID]['phones'] = csv_input[0]\n",
    "                file_meta[Audio_File_ID]['end_times'] = csv_input[1]\n",
    "                \n",
    "    with open(file_path + Audio_File_ID+'.csv', 'w', newline='') as file:\n",
    "        writer = csv.writer(file)\n",
    "        writer.writerow(csv_input)\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "    # save the updated file_meta dictionary to a CSV file\n",
    "    with open(main_path+'/DeepSpeech/data/file_meta_end_phones.csv', 'w', newline='') as csv_file:\n",
    "  \n",
    "        # Define the fieldnames of the CSV file\n",
    "        fieldnames = list(file_meta.keys())\n",
    "\n",
    "        # Create a writer object\n",
    "        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)\n",
    "\n",
    "        # Write the header row\n",
    "        writer.writeheader()\n",
    "\n",
    "        # Write the data rows\n",
    "        writer.writerow(file_meta)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                     file     accent  duration\n",
      "0  common_voice_en_533247  australia     5.332\n",
      "1  common_voice_en_195627     canada     1.956\n",
      "2   common_voice_en_19574    african     1.957\n"
     ]
    }
   ],
   "source": [
    "df_trans = pd.read_csv(main_path+'/DeepSpeech/data/test_1750_small.csv', header = None, names = ['file', 'accent', 'duration'])\n",
    "#df_trans['file'] = df_trans['file'].map(lambda x: x.split('.')[0])\n",
    "print(df_trans.head(20))\n",
    "files_list = df_trans.file.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    }
   ],
   "source": [
    "print(len(files_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Map time axis of representation to the input frames as per the convolutional layers used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_input_frame(current_frame):\n",
    "    return (current_frame - 1)*2 + 11 - 2*5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vowel_phones for German Lang\n",
    "vowel_phones=['i', 'ɪ', 'e', 'ɛ', 'a', 'ɔ', 'o', 'u', 'ʊ', 'y', 'ʏ', 'ø', 'œ', 'ə', 'ɐ', 'ɑ', 'ɛː', 'aː', 'oː', 'uː', 'yː', 'øː', 'œː', 'ɔː']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vowel_phones for English Lang\n",
    "vowel_phones = ['iy','ih','ix','ey','eh','er','ae','aa','ao','ay','aw','ah','ax','axr','ow','oy','uh','uw','ux']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Generate frame level allignments and other related data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_frame_allignment(file, input_size):\n",
    "    alligned = []\n",
    "    \n",
    "    spec_stride = 0.01\n",
    "    window_size = 0.02\n",
    "    times = file_meta[file]['end_times']\n",
    "    last_idx = 0\n",
    "    \n",
    "#     print(times[55:58])\n",
    "    for i in range(input_size):\n",
    "        frame_idx = i\n",
    "        window_start = frame_idx*spec_stride\n",
    "        window_mid = window_start + (window_size/2)\n",
    "        alligned_phone = 'na'\n",
    "        \n",
    "        \n",
    "        for j in range(len(times)):\n",
    "            \n",
    "            if (window_mid < times[j]):\n",
    "                #alligned_phone = file_meta[file]['phones'][j]\n",
    "                #print(j)\n",
    "                alligned_phone = j\n",
    "                if(j == 0 and file_meta[file]['phones'][j] =='pause'):\n",
    "                    alligned_phone = -1 # marker for start pause\n",
    "                if(j == len(times)-1 and file_meta[file]['phones'][j] == 'pause'):\n",
    "                    alligned_phone = -2 # marker for end pause\n",
    "                break\n",
    "                \n",
    "        #assert alligned_phone != 'na', \"Failed to fetch allignment\"\n",
    "        if(alligned_phone != 'na'):\n",
    "            alligned.append(alligned_phone)\n",
    "            last_idx = i\n",
    "#     pause_start = 0\n",
    "#     pause_end = len(alligned)\n",
    "#     for i in range(len(alligned)):\n",
    "#         if(alligned[i] != 'pause'):\n",
    "#             break\n",
    "#         pause_start = i\n",
    "    \n",
    "#     for i in range(len(alligned)-1,-1,-1):\n",
    "#         if(alligned[i] != 'pause'):\n",
    "#             break\n",
    "#         pause_end = i\n",
    "        \n",
    "    #print(last_idx)\n",
    "    #print(pause_start, pause_end)\n",
    "#     print(alligned)\n",
    "    allign_grouped = [x[0] for x in groupby(alligned)]\n",
    "    allign_labels = [list(x[1]) for x in groupby(alligned)]\n",
    "    #print(allign_labels)\n",
    "    allign_indices = [0]\n",
    "    for j in allign_labels:\n",
    "        allign_indices.append(allign_indices[-1] + len(j))\n",
    "    #print(allign_indices)\n",
    "    \n",
    "    return allign_grouped, allign_indices\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]\n",
      "[0, 54, 59, 68, 79, 82, 90, 98, 108, 124, 125, 135, 138, 143, 149, 155, 161, 166, 171, 178, 182, 188, 193, 199, 204, 208, 216, 224, 245]\n"
     ]
    }
   ],
   "source": [
    "labels, indices = get_frame_allignment(files_list[0],1000)\n",
    "print(labels)\n",
    "print(indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Generate allignment of representaions with phones "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_rep_labels(file,idx):\n",
    "    spec_stride = 0.01\n",
    "    window_size = 0.02\n",
    "    times = file_meta[file]['end_times']\n",
    "    frame_idx = get_input_frame(idx)\n",
    "    window_start = frame_idx*spec_stride\n",
    "    window_mid = window_start + (window_size/2)\n",
    "    alligned_phone = 'na'\n",
    "    for j in range(len(times)):\n",
    "        if (window_mid < times[j]):\n",
    "            #alligned_phone = file_meta[file]['phones'][j]\n",
    "            alligned_phone = j\n",
    "            if(j == 0 and file_meta[file]['phones'][j] =='pause'):\n",
    "                alligned_phone = -1 # marker for start pause\n",
    "            if(j == len(times)-1 and file_meta[file]['phones'][j] == 'pause'):\n",
    "                alligned_phone = -2 # marker for end pause\n",
    "            break\n",
    "    assert alligned_phone!= 'na', 'found na allignments'\n",
    "    if_vowel = False\n",
    "    if(alligned_phone >= 0):\n",
    "        if_vowel = file_meta[file]['phones'][alligned_phone] in vowel_phones\n",
    "    return alligned_phone, if_vowel\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Phone focus calculations and Neighbour Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "mixing_accents = {'us':([],[]),'indian':([],[]),'scotland':([],[]),'england':([],[]),'australia':([],[]),'canada':([],[]),'african':([],[])}\n",
    "vowel_accents = {'us':([],[]),'indian':([],[]),'scotland':([],[]),'england':([],[]),'australia':([],[]),'canada':([],[]),'african':([],[])}\n",
    "neighbours = {'us':([],[],[],[],[],[],[]),'indian':([],[],[],[],[],[],[]),'scotland':([],[],[],[],[],[],[]),'england':([],[],[],[],[],[],[]),'australia':([],[],[],[],[],[],[]),'canada':([],[],[],[],[],[],[]),'african':([],[],[],[],[],[],[])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "neighbours = {'us':([],[],[],[],[],[],[]),'indian':([],[],[],[],[],[],[]),'scotland':([],[],[],[],[],[],[]),'england':([],[],[],[],[],[],[]),'australia':([],[],[],[],[],[],[]),'canada':([],[],[],[],[],[],[]),'african':([],[],[],[],[],[],[])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# target_layer = 'rnn_4'\n",
    "target_layer = 'rnn_2'\n",
    "target_path = os.path.join(contribution_path,target_layer)\n",
    "# print(target_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rep_type='conv'\n",
    "phones_path=main_path+'/DeepSpeech/data/Contribution/MCV/{}/'.format(rep_type)\n",
    "\n",
    "aaaa = len(glob.glob1(phones_path,\"*.npy\"))\n",
    "aaaa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/mmm2050/QU_DFKI_Thesis/Experimentation/ASR_Accent_Analysis_DeDeepSpeech/data/Contribution/MCV/conv/'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phones_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['common_voice_en_195627_3_conv.npy',\n",
       " 'common_voice_en_533247_0_conv.npy',\n",
       " 'common_voice_en_19574_3_conv.npy',\n",
       " 'common_voice_en_533247_3_conv.npy',\n",
       " 'common_voice_en_195627_2_conv.npy',\n",
       " 'common_voice_en_533247_2_conv.npy',\n",
       " 'common_voice_en_19574_2_conv.npy',\n",
       " 'common_voice_en_195627_1_conv.npy',\n",
       " 'common_voice_en_19574_0_conv.npy',\n",
       " 'common_voice_en_533247_1_conv.npy',\n",
       " 'common_voice_en_195627_0_conv.npy',\n",
       " 'common_voice_en_19574_1_conv.npy']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phones_path=main_path+'/DeepSpeech/data/Contribution/MCV/conv/'\n",
    "aa = glob.glob1(phones_path,\"*.npy\")\n",
    "aa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in files_list:\n",
    "    #file = 'common_voice_en_533247'\n",
    "    #print(file)\n",
    "\n",
    "#     print(file_meta[file]['end_times'])\n",
    "#     print(file_meta[file]['phones'])\n",
    "    labels, indices = get_frame_allignment(file,1500)\n",
    "#     print(labels)\n",
    "#     print(len(indices))\n",
    "    desired_files = glob.glob(target_path +'/{}*.npy'.format(file))\n",
    "    #print(desired_files)\n",
    "    for f in desired_files:\n",
    "        try:\n",
    "        #print(f)\n",
    "#             print(f)\n",
    "            index = f.split('_')[-2]\n",
    "            if('rnn' in target_layer):\n",
    "                index = f.split('_')[-3]\n",
    "                \n",
    "            #print(index)\n",
    "\n",
    "            #print(index)\n",
    "            out, if_vowel = get_rep_labels(file,int(index))\n",
    "            if(out<0):\n",
    "                continue\n",
    "            out_id = labels.index(out)\n",
    "    #         print(out)\n",
    "    #         print(labels)\n",
    "    #         print(indices)\n",
    "            arr = np.load(f)\n",
    "            if(np.sum(arr) == 0):\n",
    "                #print('encountered all 0')\n",
    "                continue\n",
    "            #plt.plot(arr)\n",
    "            #print(arr[:50])\n",
    "            #print(np.sum(arr))\n",
    "    #         print(len(arr))\n",
    "    #         print(arr[indices[out]-20:indices[out + 1]+20])\n",
    "            sliced = arr[indices[out_id]:indices[out_id + 1]]\n",
    "            contr = np.sum(sliced)*100\n",
    "            if(np.isnan(contr)):\n",
    "                continue\n",
    "            \n",
    "            #print(contr)\n",
    "\n",
    "            all_contr = []\n",
    "            for i in range(len(labels)):\n",
    "\n",
    "    #             if(labels[i] <0):\n",
    "    #                 continue\n",
    "    # #             print(i)\n",
    "                all_contr.append(np.sum(arr[indices[i]:indices[i + 1]]))\n",
    "            max_idx = np.argmax(np.asarray(all_contr))\n",
    "\n",
    "            cond = (max_idx == out_id)\n",
    "#             print(max_idx,out_id,all_contr[max_idx]*100)\n",
    "            #print(contr, cond, max_idx, out_id)\n",
    "            accent = file_meta[file]['accent']\n",
    "            mixing_accents[accent][0].append(contr)\n",
    "            mixing_accents[accent][1].append(cond)\n",
    "            if(if_vowel):\n",
    "                vowel_accents[accent][0].append(contr)\n",
    "                vowel_accents[accent][1].append(cond)\n",
    "        except:\n",
    "            #print('failed for file:',file,index)\n",
    "            continue\n",
    "#         break\n",
    "#     break\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Contr:\n",
    "    def __init__(self, arr):\n",
    "        self.arr = arr\n",
    "        self.len = len(arr)\n",
    "        #print(arr)\n",
    "        #print('done')\n",
    "    def fetch(self,idx):\n",
    "        #print(self.arr)\n",
    "        if(idx < 0 or idx >= self.len):\n",
    "            #print('enc')\n",
    "            return 0\n",
    "        else:\n",
    "            #print('nor')\n",
    "            return self.arr[idx]\n",
    "    def fetch_range(self,r ):\n",
    "        sum = 0.0\n",
    "        (start, stop) = r\n",
    "        for i in range(start,stop +1):\n",
    "            sum += self.fetch(i)\n",
    "        return sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in files_list:\n",
    "    #file = 'common_voice_en_55029'\n",
    "    #print(file)\n",
    "\n",
    "#     print(file_meta[file]['end_times'])\n",
    "#     print(file_meta[file]['phones'])\n",
    "    labels, indices = get_frame_allignment(file,1500)\n",
    "#     print(labels)\n",
    "#     print(len(indices))\n",
    "    desired_files = glob.glob(target_path +'/{}*.npy'.format(file))\n",
    "    #print(desired_files)\n",
    "    for f in desired_files:\n",
    "        try:\n",
    "            #print(f)\n",
    "    #             print(f)\n",
    "            index = f.split('_')[-2]\n",
    "            if('rnn' in target_layer):\n",
    "                index = f.split('_')[-3]\n",
    "\n",
    "            #print(index)\n",
    "\n",
    "            #print(index)\n",
    "            out, if_vowel = get_rep_labels(file,int(index))\n",
    "            if(out<0):\n",
    "                continue\n",
    "            out_id = labels.index(out)\n",
    "    #         print(out)\n",
    "    #         print(labels)\n",
    "    #         print(indices)\n",
    "            arr = np.load(f)\n",
    "            if(np.sum(arr) == 0):\n",
    "                #print('encountered all 0')\n",
    "                continue\n",
    "            #plt.plot(arr)\n",
    "            #print(arr[:50])\n",
    "            #print(np.sum(arr))\n",
    "    #         print(len(arr))\n",
    "    #         print(arr[indices[out]-20:indices[out + 1]+20])\n",
    "            sliced = arr[indices[out_id]:indices[out_id + 1]]\n",
    "            contr = np.sum(sliced)*100\n",
    "            if(np.isnan(contr)):\n",
    "                continue\n",
    "\n",
    "            #print(contr)\n",
    "\n",
    "            all_contr = []\n",
    "            for i in range(len(labels)):\n",
    "\n",
    "    #             if(labels[i] <0):\n",
    "    #                 continue\n",
    "    # #             print(i)\n",
    "                all_contr.append(np.sum(arr[indices[i]:indices[i + 1]]))\n",
    "\n",
    "            max_idx = np.argmax(np.asarray(all_contr))\n",
    "            neigh_contr = Contr(all_contr)\n",
    "\n",
    "            #print(neigh_contr.fetch(0))\n",
    "            cond = (max_idx == out_id)\n",
    "    #             print(max_idx,out_id,all_contr[max_idx]*100)\n",
    "            #print(contr, cond, max_idx, out_id)\n",
    "            accent = file_meta[file]['accent']\n",
    "            #accent = 'timit'\n",
    "    #         mixing_timit[accent][0].append(contr)\n",
    "    #         mixing_timit[accent][1].append(cond)\n",
    "            neighbours[accent][0].append(neigh_contr.fetch(out_id -1) + neigh_contr.fetch(out_id +1))\n",
    "            neighbours[accent][1].append(neigh_contr.fetch(out_id -2) + neigh_contr.fetch(out_id +2))\n",
    "            neighbours[accent][2].append(neigh_contr.fetch(out_id -3) + neigh_contr.fetch(out_id +3))\n",
    "            neighbours[accent][3].append(neigh_contr.fetch_range((out_id - 5, out_id -4)) + neigh_contr.fetch_range((out_id +4, out_id+5)))\n",
    "            neighbours[accent][4].append(neigh_contr.fetch_range((out_id - 8, out_id -6)) + neigh_contr.fetch_range((out_id +6, out_id+8)))\n",
    "            neighbours[accent][5].append(neigh_contr.fetch_range((out_id - 11, out_id -9)) + neigh_contr.fetch_range((out_id +9, out_id+11)))\n",
    "            neighbours[accent][6].append(neigh_contr.fetch_range((out_id - 100, out_id -12)) + neigh_contr.fetch_range((out_id +12, out_id+100)))\n",
    "\n",
    "\n",
    "#             if(if_vowel):\n",
    "#                 vowels_timit[accent][0].append(contr)\n",
    "#                 vowels_timit[accent][1].append(cond)\n",
    "        except:\n",
    "            #print('failed for file:',file,index)\n",
    "            continue\n",
    "#         break\n",
    "#     break\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Phone focus and binary phone focus (calculated one layer at a time), shown here for SPEC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "australia\n",
      "33.487499411392015 25.95126131362894 70.73170731707317\n",
      "canada\n",
      "30.215999327691986 15.286055812193885 69.1358024691358\n",
      "african\n",
      "32.34798035726113 26.552973972379252 61.594202898550726\n"
     ]
    }
   ],
   "source": [
    "test_En_accents=['australia','canada','african']\n",
    "# for a in mixing_accents.keys():\n",
    "for a in test_En_accents:\n",
    "    try:\n",
    "        print(a)\n",
    "#         \n",
    "        contr_arr = np.asarray(mixing_accents[a][0])\n",
    "#         print(mixing_accents[a][1])\n",
    "#         print(mixing_accents[a][0])\n",
    "#         break\n",
    "        print(contr_arr.mean(),contr_arr.std(),100.0*sum(mixing_accents[a][1])/(len(mixing_accents[a][1])))\n",
    "        # print('No God but Allah')\n",
    "      \n",
    "    except:\n",
    "        continue\n",
    "# for a in vowel_accents.keys():\n",
    "#     try:\n",
    "#         print(a)\n",
    "# #         \n",
    "#         contr_arr = np.asarray(vowel_accents[a][0])\n",
    "# #         print(mixing_accents[a][1])\n",
    "# #         print(mixing_accents[a][0])\n",
    "# #         break\n",
    "#         print(contr_arr.mean(),contr_arr.std(),100.0*sum(vowel_accents[a][1])/(len(vowel_accents[a][1])))\n",
    "#     except:\n",
    "#         continue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Neighbour Analysis values for layer RNN_4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "australia\n",
      "[16.868573874689456, 9.769022675246243, 5.179731914682723, 6.084669383273013, 5.55994719638616, 3.269251322419178, 11.274998614503492]\n",
      "canada\n",
      "[25.957854368068556, 9.620045498013496, 4.9559096219363035, 5.5821913560381, 6.359451801018835, 3.9382732239764553, 8.28623693784193]\n",
      "african\n",
      "[19.758450688686278, 10.609218771771891, 7.070050670164704, 7.39141668908406, 6.55961144100571, 4.177824337028247, 8.68645223014502]\n"
     ]
    }
   ],
   "source": [
    "test_En_accents=['australia','canada','african']\n",
    "# for a in mixing_accents.keys():\n",
    "for a in test_En_accents:\n",
    "    try:\n",
    "        print(a)\n",
    "#         \n",
    "        #contr_arr = np.asarray(mixing_accents[a][0])\n",
    "#         print(mixing_accents[a][1])\n",
    "#         print(mixing_accents[a][0])\n",
    "#         break\n",
    "        #print(neighbours[a][0])\n",
    "#         print(100*np.asarray(neighbours[a][0]).mean())\n",
    "        arr = []\n",
    "        for i in range(len(neighbours[a])):\n",
    "            arr.append(100*np.asarray(neighbours[a][i]).mean())\n",
    "        print(arr)\n",
    "        #print(contr_arr.mean(),contr_arr.std(),100.0*sum(mixing_timit[a][1])/(len(mixing_timit[a][1])))\n",
    "    except:\n",
    "        continue\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conv \n",
    "# us\n",
    "# [32.7407032251358, 4.129967677019115, 0.5069611499109884, 0.07540823335304898, 0.0020524582412921005, 0.0, 0.0]\n",
    "# indian\n",
    "# [30.90875744819641, 4.187893504497704, 0.6535981730305066, 0.238160239311137, 0.046958024210296455, 0.003738099867418384, 0.0002974208116719145]\n",
    "# scotland\n",
    "# [32.13910460472107, 4.590714934856013, 0.7024039413508787, 0.17588622736358267, 0.02393076658670436, 0.0026134167411263247, 0.0]\n",
    "# england\n",
    "# [32.66556247854946, 4.368573959888614, 0.6079838453871551, 0.11953031163447808, 0.01943777885757124, 0.002580319954526009, 0.0004979741529129851]\n",
    "# australia\n",
    "# [32.34032988548279, 4.472068407601206, 0.632637141106141, 0.11507221019796406, 0.009235619216210154, 1.952351730773e-05, 0.0]\n",
    "# canada\n",
    "# [33.539554476737976, 4.574416147792803, 0.5713643466006877, 0.07515107721887157, 0.007279535102926467, 0.001223377025871137, 0.0]\n",
    "# african\n",
    "# [31.233730379140706, 4.009490048996913, 0.5022208455794831, 0.07889555733309296, 0.00871781443660488, 0.0008204423270378301, 6.012520126919877e-05]\n",
    "\n",
    "# rnn_0\n",
    "# us\n",
    "# [27.49955654144287, 5.13346373603394, 2.0309174663958474, 2.6767877794440995, 3.0200369344228926, 2.362625394259192, 7.852535240260654]\n",
    "# indian\n",
    "# [26.19927227497101, 5.090281970997703, 2.077755862808442, 2.7060146116646786, 2.9382837094459093, 2.2580741789313943, 7.560508918494634]\n",
    "# scotland\n",
    "# [26.919716596603394, 5.392323765776727, 2.1341611514668424, 2.674973549457239, 2.9833333826550628, 2.3151264046490434, 8.07114260450446]\n",
    "# england\n",
    "# [27.37573602958441, 5.295955654012483, 2.1334719217250915, 2.7150068105729908, 3.0929771560113015, 2.4152857833745416, 7.717478458531311]\n",
    "# australia\n",
    "# [27.247267961502075, 5.34691052375551, 2.105283901850338, 2.673661797066982, 3.067990261786627, 2.3905664178446235, 7.4642531857985]\n",
    "# canada\n",
    "# [28.169283270835876, 5.448183687807504, 2.077020231947467, 2.636450567206863, 3.0098093966374746, 2.369359405710461, 7.507328188736076]\n",
    "# african\n",
    "# [26.15024838869291, 4.877831960924951, 1.9293778762854086, 2.5777538785687097, 2.9440899393857025, 2.303586629368907, 8.30029107582431]\n",
    "\n",
    "# rnn_1\n",
    "# us\n",
    "# [23.962603509426117, 8.507628458622953, 4.465992478354428, 5.71455936704617, 6.195458850237518, 4.829749567876604, 16.16509212889644]\n",
    "# indian\n",
    "# [23.195047676563263, 8.404100489357473, 4.470128897364782, 5.671139707643684, 6.008702689557492, 4.620739478180099, 15.721930947546001]\n",
    "# scotland\n",
    "# [23.04048240184784, 8.462198564993297, 4.534059670150146, 5.696610933284059, 6.151703288604992, 4.786315666033271, 16.790108252422357]\n",
    "# england\n",
    "# [23.631045862128857, 8.5951688668769, 4.632277741647281, 5.784687362959637, 6.338151672549959, 4.936726220444522, 16.040685559373344]\n",
    "# australia\n",
    "# [23.752808570861816, 8.661556305105101, 4.572080661245361, 5.7502741717005055, 6.307751578932401, 4.898818396068181, 15.481466421210937]\n",
    "# canada\n",
    "# [24.332773685455322, 8.688939460844866, 4.547470026116145, 5.6507561064439, 6.15434375223156, 4.845393838018302, 15.452406634770167]\n",
    "# african\n",
    "# [22.747576780279, 8.000965653438755, 4.25131166990766, 5.47551236200026, 6.0265592081826975, 4.722378272694236, 17.231608823099382]\n",
    "\n",
    "# rnn_2\n",
    "# us\n",
    "# [24.472643435001373, 11.416035009853484, 6.433550829242235, 7.172085362697694, 6.3121468964634735, 4.558863821094617, 14.998742030734071]\n",
    "# indian\n",
    "# [23.51335436105728, 11.387106717103787, 6.649511262522023, 7.51940392086301, 6.3137742982306, 4.358723318068517, 14.450793554558317]\n",
    "# scotland\n",
    "# [23.225121200084686, 11.172301102451147, 6.492206765380377, 7.298659402172287, 6.401033581395596, 4.550037674420537, 15.69172966370581]\n",
    "# england\n",
    "# [23.913722893040283, 11.459426143523336, 6.633741471145262, 7.348491074447801, 6.48838323695089, 4.668961221382211, 14.981185589184959]\n",
    "# australia\n",
    "# [24.096539616584778, 11.600811539104825, 6.5722590834003265, 7.360531804260481, 6.491923510091799, 4.633371833046721, 14.393180537147854]\n",
    "# canada\n",
    "# [24.683956801891327, 11.572113914395796, 6.535514317857894, 7.151745245651895, 6.264582652847311, 4.568181247712387, 14.349837376059055]\n",
    "# african\n",
    "# [23.278117922402455, 10.784819814874142, 6.167853194400013, 6.9110016395469005, 6.130243824584725, 4.448534439984776, 16.056985357788058]\n",
    "\n",
    "# rnn_3\n",
    "# us\n",
    "# [23.23090434074402, 12.174146764277422, 7.504013566115788, 8.723696850925187, 7.227962272073496, 4.761639574511886, 14.509863650139298]\n",
    "# indian\n",
    "# [21.902601420879364, 12.02527095338879, 7.787259836802122, 9.424352349689228, 7.673399363880054, 4.747935228341197, 13.992496343298566]\n",
    "# scotland\n",
    "# [21.891167759895325, 11.795758316695142, 7.507693259656952, 8.898633270182273, 7.511580442325496, 4.847434087985344, 15.216503226337569]\n",
    "# england\n",
    "# [22.57687249452284, 12.14707995605362, 7.730428943421168, 9.010231605380561, 7.541451811973519, 4.898843390369724, 14.478454061859688]\n",
    "# australia\n",
    "# [22.68569767475128, 12.253803979128968, 7.636378218902687, 9.0469665974413, 7.621676064515722, 4.872514438164382, 13.939816792289179]\n",
    "# canada\n",
    "# [23.474161326885223, 12.297419011472826, 7.59384786153115, 8.712447004216948, 7.161918183838477, 4.73416186686994, 13.869042442307574]\n",
    "# african\n",
    "# [22.116470338617592, 11.47845570845182, 7.220993539734151, 8.530101589118326, 7.0719255800387915, 4.653464676917603, 15.48961048674086]\n",
    "\n",
    "# rnn_4\n",
    "# us\n",
    "# [22.638480365276337, 12.542382574099609, 7.952615837516744, 9.385792697883067, 7.614086639646633, 4.851851712807524, 14.444850730930911]\n",
    "# indian\n",
    "# [21.280723810195923, 12.305844389243482, 8.195414496937591, 10.123701281435645, 8.182893904021277, 4.914469927645419, 13.960731343402333]\n",
    "# scotland\n",
    "# [21.29308432340622, 12.132624955503704, 7.9527269285730995, 9.562700520589063, 7.945317385160462, 4.971957288337058, 15.219619409445926]\n",
    "# england\n",
    "# [21.94535266152243, 12.496599320776916, 8.170664550136404, 9.697629917171719, 7.984599989060589, 5.00539373145459, 14.395554327181582]\n",
    "# australia\n",
    "# [22.04861491918564, 12.597264878502923, 8.086613376326612, 9.744694454159763, 8.074721421988148, 4.995766007349288, 13.901542000918287]\n",
    "# canada\n",
    "# [22.8696346282959, 12.696123461599603, 8.067566000259314, 9.395388820129348, 7.578003661423454, 4.830882189746786, 13.786903177871535]\n",
    "# african\n",
    "# [21.585925028024786, 11.85953984030524, 7.679363588716809, 9.199444571711155, 7.494925847285976, 4.779391716000584, 15.443010087585678]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (308300810.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  Cell \u001b[0;32mIn[171], line 1\u001b[0;36m\u001b[0m\n\u001b[0;31m    failed for file: common_voice_en_117181 1 22\u001b[0m\n\u001b[0m           ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "failed for file: common_voice_en_117181 1 22\n",
    "failed for file: common_voice_en_16666058 1 25"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}