Text_processing_and_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iNocuCQiIYQ8"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models.phrases import Phrases, Phraser\n",
    "import json\n",
    "from nltk import RegexpTokenizer\n",
    "import time\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "import random\n",
    "import multiprocessing\n",
    "from nltk import RegexpTokenizer\n",
    "import copy\n",
    "import pickle\n",
    "multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_tSvCm6HI-Xh"
   },
   "outputs": [],
   "source": [
    "image_path = '/ImageCap/imagesB/flickr30k-images/'\n",
    "text_path = '/ImageCap/captions.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "szLVVS_2JFuS"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(text_path ,delimiter='|', skipinitialspace=True) #in this csv, | is being used as delimiter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HIedKiVqJIk-"
   },
   "outputs": [],
   "source": [
    "groups = [b for a,b in df.groupby('image_name')] #Seperating rows into groups as per image name\n",
    "random.shuffle(groups) #applying shuffle operation group-wise\n",
    "df = pd.concat(groups).reset_index(drop=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3M9j-zEVJjsE"
   },
   "outputs": [],
   "source": [
    "image_name_list = list(set(df['image_name'])) #obtaining unique instance name of each image\n",
    "image_path_list = list(map(lambda arg: image_path + arg, image_name_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "uMgP8JZmJ4Fl",
    "outputId": "e55704dc-db18-4513-96fc-acc7f2ae24bf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "image_name                                           3296226598.jpg\n",
       "comment_number                                                    0\n",
       "comment           A man in a black leather jacket walks next to ...\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yxQnG1dYMTQu"
   },
   "outputs": [],
   "source": [
    "df['comment'] = df['comment'].apply(lambda arg: '<start> ' + str(arg) + ' <end>') #putting start and end tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "L6dH48NfNSbC",
    "outputId": "3e5baf06-91d3-4dff-ff23-c3bbae41e194"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<start> A man in a black leather jacket walks next to a white brick building in a big city . <end>'"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[0]['comment']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dO9zYbrYNY1a"
   },
   "outputs": [],
   "source": [
    "tokenizer = RegexpTokenizer(r'\\w+|<start>|<end>') #tokenizing using only alphanumeric tokens and start, end tokens\n",
    "df['tokens'] = df['comment'].apply(tokenizer.tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 442
    },
    "colab_type": "code",
    "id": "LW5DrGTtOhAR",
    "outputId": "eb452f96-e9d4-471e-81a0-b448817bd3b9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<start>',\n",
       " 'A',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'black',\n",
       " 'leather',\n",
       " 'jacket',\n",
       " 'walks',\n",
       " 'next',\n",
       " 'to',\n",
       " 'a',\n",
       " 'white',\n",
       " 'brick',\n",
       " 'building',\n",
       " 'in',\n",
       " 'a',\n",
       " 'big',\n",
       " 'city',\n",
       " '<end>']"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[0]['tokens']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y5xg2_40OwlA"
   },
   "outputs": [],
   "source": [
    "#Finding max length for padding\n",
    "len_ = df['tokens'].apply(lambda arg: len(arg))\n",
    "max_len = max(len_) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KFFUJCnIQSyZ"
   },
   "outputs": [],
   "source": [
    "#padding to max_len\n",
    "def pad(arg):\n",
    "  len_ = len(arg)\n",
    "  len_ = max_len - len_\n",
    "  lst = copy.deepcopy(arg)\n",
    "  for i in np.arange(0,len_):\n",
    "    lst.append('_')\n",
    "  return lst\n",
    "\n",
    "df['tokens_pad'] = df['tokens'].apply(pad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "IdkcCjVhQUgd",
    "outputId": "8254e96d-9cd1-4540-e10a-3e175cdcd2df"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<start>',\n",
       " 'A',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'black',\n",
       " 'leather',\n",
       " 'jacket',\n",
       " 'walks',\n",
       " 'next',\n",
       " 'to',\n",
       " 'a',\n",
       " 'white',\n",
       " 'brick',\n",
       " 'building',\n",
       " 'in',\n",
       " 'a',\n",
       " 'big',\n",
       " 'city',\n",
       " '<end>',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_',\n",
       " '_']"
      ]
     },
     "execution_count": 11,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[0]['tokens_pad']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TQQVC_dV3d2I",
    "outputId": "2cf4e02c-8c1e-402f-9879-2b0685c6bc52"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131.58676886558533\n"
     ]
    }
   ],
   "source": [
    "#Training our custom word2vec model (its imp to do this because google's w2v lacks stopwords and <start>, <end> tokens)\n",
    "t=time.time()\n",
    "obj_w2v=Word2Vec(sentences=df['tokens'],min_count=1,window=2,size=300,sample=6e-5, alpha=0.03, min_alpha=0.0007, negative=20,workers=2,iter=30)\n",
    "print(time.time()-t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q0fUk2iP3d2O",
    "outputId": "7c2a006f-1ddd-4d4b-9095-7d2ec8d86454"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dell\\Anaconda3\\lib\\site-packages\\smart_open\\smart_open_lib.py:402: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
      "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "144.9913194179535\n"
     ]
    }
   ],
   "source": [
    "#Loading Google's pretrainind w2v model\n",
    "t=time.time()\n",
    "word2vec_path = \"/ImageCap/misc/GoogleNews-vectors-negative300.bin.gz\"\n",
    "w2v_g = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=True)\n",
    "print(time.time()-t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OSk5s2Tb3d2U",
    "outputId": "8ada14be-1bca-448b-a490-cd762899fe9c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "999.7349133491516\n"
     ]
    }
   ],
   "source": [
    "#updating our local model's vocab with google's w2v's vocab\n",
    "t=time.time()\n",
    "obj_w2v.build_vocab([list(w2v_g.vocab.keys())],update=True)\n",
    "print(time.time()-t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O5oQeV043d2d",
    "outputId": "6825808d-cc77-4d50-dbc0-d1785d504246"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "170.2057385444641\n"
     ]
    }
   ],
   "source": [
    "t=time.time()\n",
    "obj_w2v.intersect_word2vec_format('/ImageCap/misc/GoogleNews-vectors-negative300.bin.gz',lockf=1.0, binary=True)\n",
    "#this function will look for words common in our vocab \n",
    "#and in google's vocab and change the weight of such words to that in google's weight vector. \n",
    "#Here we have set lockf=1 so that in further training weights of all the\n",
    "#words are optimized\n",
    "\n",
    "print(time.time()-t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 88
    },
    "colab_type": "code",
    "id": "HvXadU-DXahI",
    "outputId": "cbf95643-f19e-45c4-c28e-5d32d43a6331"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Dell\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: DeprecationWarning: Call to deprecated `iter` (Attribute will be removed in 4.0.0, use self.epochs instead).\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "627.3765051364899\n"
     ]
    }
   ],
   "source": [
    "t=time.time()\n",
    "obj_w2v.train(df['tokens'],total_examples=len(df),epochs=obj_w2v.iter)#we are training again so as to get weights adjusted\n",
    "#This retraining is important so as to avoid clash of embedding space of google's w2v and our custom w2v\n",
    "print(time.time()-t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hpyuh9XpjzNX"
   },
   "outputs": [],
   "source": [
    "obj_w2v.save('/ImageCap/w2v_imageCap.kv') #This will save entire model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DAOzaxOD3d2p"
   },
   "outputs": [],
   "source": [
    "obj_w2v.wv.save_word2vec_format('/ImageCap/w2v_imageCap.bin', binary=True) #This will only save matrix / vocab of model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4fUkUeR_3d2t"
   },
   "outputs": [],
   "source": [
    "word2vec = gensim.models.KeyedVectors.load_word2vec_format('/ImageCap/w2v_imageCap.bin', binary=True,unicode_errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now we will create a new vocab of most used tokens and then later map them to their w2v embeddings\n",
    "vec=Tokenizer()\n",
    "vec.fit_on_texts(df['tokens'])\n",
    "l=len((vec.word_index))\n",
    "#print(l)\n",
    "#print((vec.word_index))#dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Here we are generating a matrix that contains embedding of each word in our vocabulary\n",
    "r=0\n",
    "M=(np.zeros((l,300)))\n",
    "#d=pd.DataFrame()\n",
    "for el in vec.word_index:\n",
    "  try:\n",
    "    M[r,:]=word2vec[str(el)]#put entire word vector for el in rth row\n",
    "  except:\n",
    "    M[r,:]=np.zeros((1,300)) #in case of an unknown word, simply put zero vector\n",
    "    #print('error for ',' ',el)\n",
    "  #d=d.append(pd.DataFrame(data={'row':[r], 'word': [el]}))# storing the mapping of row to word\n",
    "  r=r+1\n",
    "\n",
    "#print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "J23VDXIs3d26",
    "outputId": "3c3df41a-30da-46f1-c9d7-f1fce7a05574"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18289, 300)"
      ]
     },
     "execution_count": 36,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YDOVuazNwGoV"
   },
   "outputs": [],
   "source": [
    "#pad token '_' has been dealt seperately because it might have affected the word2vec training if used earlier\n",
    "vec.word_index['_'] = 0 #adding 0 for padding\n",
    "vec.index_word[0] = '_'\n",
    "M = np.vstack((np.zeros((1,300)),M)) #putting zero vector at 0th row for padding\n",
    "seq = vec.texts_to_sequences(df['tokens_pad']) #converting tokens to corrosponding indicies so that they can be\n",
    "                                               #they can be used for mapping by embedding layer during training\n",
    "seq_vec = np.array(seq).astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_Whtos1Zwdit"
   },
   "outputs": [],
   "source": [
    "df.to_csv(r'/ImageCap/captions_pros.csv',index=None,header=True) # all further mappings will be as per this csv\n",
    "np.save('/ImageCap/embedB.npy',M)\n",
    "np.save('/ImageCap/caption_vec.npy',seq_vec)\n",
    "\n",
    "with open('/ImageCap/word_ind_map.pkl', 'wb') as f:\n",
    "  pickle.dump(vec.word_index, f, pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open('/ImageCap/ind_word_map.pkl', 'wb') as f:\n",
    "  pickle.dump(vec.index_word, f, pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Text_processing_and_embedding_local.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}