data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(200000, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is nothing special about this streamer. ...</td>\n",
       "      <td>Mediocre</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "      <td>Duffer. \"Bo\"</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For starters, I did not receive the keyboard t...</td>\n",
       "      <td>Can't Use</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "      <td>Duffer. \"Bo\"</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>My home is located about half way between wher...</td>\n",
       "      <td>So Far, So Good</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "      <td>Duffer. \"Bo\"</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In the box is the quite attractive unit, a fil...</td>\n",
       "      <td>Almost</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "      <td>Duffer. \"Bo\"</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I have reviewed this previously but it shows a...</td>\n",
       "      <td>Repeat Review</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "      <td>Duffer. \"Bo\"</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0                1  \\\n",
       "0  There is nothing special about this streamer. ...         Mediocre   \n",
       "1  For starters, I did not receive the keyboard t...        Can't Use   \n",
       "2  My home is located about half way between wher...  So Far, So Good   \n",
       "3  In the box is the quite attractive unit, a fil...           Almost   \n",
       "4  I have reviewed this previously but it shows a...    Repeat Review   \n",
       "\n",
       "                2             3  4  \n",
       "0  A100WO06OQR8BQ  Duffer. \"Bo\"  2  \n",
       "1  A100WO06OQR8BQ  Duffer. \"Bo\"  3  \n",
       "2  A100WO06OQR8BQ  Duffer. \"Bo\"  4  \n",
       "3  A100WO06OQR8BQ  Duffer. \"Bo\"  5  \n",
       "4  A100WO06OQR8BQ  Duffer. \"Bo\"  6  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import pymysql \n",
    "from functools import reduce\n",
    "import numpy as np\n",
    "import textstat\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import sent_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import random\n",
    "\n",
    "con = pymysql.connect(host='localhost', user='root', passwd='qwer', db='amazon', port = 3306) \n",
    "cur = con.cursor()\n",
    "\n",
    "def read_table(cur, sql_order): # sql_order is a string\n",
    "     try:\n",
    "         cur.execute(sql_order) \n",
    "         data  = cur.fetchall(  )\n",
    "         frame = pd.DataFrame(list(data))\n",
    "     except: #, e:\n",
    "         frame = pd.DataFrame()\n",
    "         # print e\n",
    "         # continue \n",
    "     return frame\n",
    " \n",
    "df_train = read_table(cur,'select * from final_train200') #35000 line－>80k line－>170k\n",
    "df_val = read_table(cur,'select * from final_val200') #5000 line->5k line->10k\n",
    "df_test = read_table(cur,'select * from final_test200') #10000 line->10k line->20k\n",
    "\n",
    "cur.close()\n",
    "con.close()\n",
    "df = pd.concat([df_train,df_val,df_test], axis=0)\n",
    "df = pd.DataFrame(df)  #df is turple, need to convert to dataframe\n",
    "print(df.shape)\n",
    "pd.set_option('max_colwidth',50)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1002\n",
      "1000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>custom_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is nothing special about this streamer. ...</td>\n",
       "      <td>Mediocre</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For starters, I did not receive the keyboard t...</td>\n",
       "      <td>Can't Use</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>My home is located about half way between wher...</td>\n",
       "      <td>So Far, So Good</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In the box is the quite attractive unit, a fil...</td>\n",
       "      <td>Almost</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I have reviewed this previously but it shows a...</td>\n",
       "      <td>Repeat Review</td>\n",
       "      <td>A100WO06OQR8BQ</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         description            title  \\\n",
       "0  There is nothing special about this streamer. ...         Mediocre   \n",
       "1  For starters, I did not receive the keyboard t...        Can't Use   \n",
       "2  My home is located about half way between wher...  So Far, So Good   \n",
       "3  In the box is the quite attractive unit, a fil...           Almost   \n",
       "4  I have reviewed this previously but it shows a...    Repeat Review   \n",
       "\n",
       "        custom_id  \n",
       "0  A100WO06OQR8BQ  \n",
       "1  A100WO06OQR8BQ  \n",
       "2  A100WO06OQR8BQ  \n",
       "3  A100WO06OQR8BQ  \n",
       "4  A100WO06OQR8BQ  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## customer name has some consistency problem with mysql and pandas.\n",
    "## since in mysql, we choose 1 id per name, thus id can also be used as the single identity as name.\n",
    "## we will choose custom_id as the Y label, drop custom name.\n",
    "df = df.rename(columns={0:'description',1:'title',2:'custom_id',3:'custom_name', 4:'row_num'})\n",
    "print(df['custom_name'].nunique())\n",
    "print(df['custom_id'].nunique())\n",
    "df = df.drop(['custom_name','row_num'], axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>custom_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is nothing special about this streamer. ...</td>\n",
       "      <td>Mediocre</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For starters, I did not receive the keyboard t...</td>\n",
       "      <td>Can't Use</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>My home is located about half way between wher...</td>\n",
       "      <td>So Far, So Good</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In the box is the quite attractive unit, a fil...</td>\n",
       "      <td>Almost</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I have reviewed this previously but it shows a...</td>\n",
       "      <td>Repeat Review</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         description            title  \\\n",
       "0  There is nothing special about this streamer. ...         Mediocre   \n",
       "1  For starters, I did not receive the keyboard t...        Can't Use   \n",
       "2  My home is located about half way between wher...  So Far, So Good   \n",
       "3  In the box is the quite attractive unit, a fil...           Almost   \n",
       "4  I have reviewed this previously but it shows a...    Repeat Review   \n",
       "\n",
       "   custom_id  \n",
       "0          0  \n",
       "1          0  \n",
       "2          0  \n",
       "3          0  \n",
       "4          0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## encode custom_id to int\n",
    "def get_series_ids(x):\n",
    "    '''Function returns a pandas series consisting of ids, \n",
    "       corresponding to objects in input pandas series x\n",
    "       Example: \n",
    "       get_series_ids(pd.Series(['a','a','b','b','c'])) \n",
    "       returns Series([0,0,1,1,2], dtype=int)'''\n",
    "\n",
    "    values = np.unique(x)\n",
    "    values2nums = dict(zip(values,range(len(values))))\n",
    "    return x.replace(values2nums)\n",
    "df['custom_id'] = get_series_ids(df['custom_id'])   #use int encoding for custome_id\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(200000, 9)\n",
      "(200000, 9)\n",
      "(200000, 21)\n"
     ]
    }
   ],
   "source": [
    "##extract pos of TAG feature\n",
    "def SyntacticFeatures(chapters):\n",
    "    \"\"\"\n",
    "    Extract feature vector for part of speech frequencies\n",
    "    \"\"\"\n",
    "    def token_to_pos(ch):\n",
    "        tokens = nltk.word_tokenize(ch)\n",
    "        return [p[1] for p in nltk.pos_tag(tokens)]\n",
    "\n",
    "    chapters_pos = [token_to_pos(ch) for ch in chapters]\n",
    "    pos_list = ['NN', 'NNP', 'DT', 'IN', 'JJ', 'NNS','UH','PDT','MD']\n",
    "    fvs_syntax = np.array([[ch.count(pos) for pos in pos_list]\n",
    "                           for ch in chapters_pos]).astype(np.float64)\n",
    "\n",
    "    # normalise by dividing each row by number of tokens in the chapter\n",
    "    fvs_syntax /= np.c_[np.array([len(ch) for ch in chapters_pos])]\n",
    "\n",
    "    return fvs_syntax, len(pos_list)\n",
    "\n",
    "chapters = df['description'].values.tolist()\n",
    "desc_syntax_statis,tag_num = SyntacticFeatures(chapters)\n",
    "print(desc_syntax_statis.shape)\n",
    "desc_syntax_statis =  pd.DataFrame(desc_syntax_statis)\n",
    "desc_syntax_statis.columns = ['desc_tag_'+str(i+1) for i in range(tag_num)]\n",
    "\n",
    "chapters = df['title'].values.tolist()\n",
    "title_syntax_statis,tag_num  = SyntacticFeatures(chapters)\n",
    "print(title_syntax_statis.shape)\n",
    "title_syntax_statis =  pd.DataFrame(title_syntax_statis)\n",
    "title_syntax_statis.columns = ['title_tag_'+str(i+1) for i in range(tag_num)]\n",
    "\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "df = pd.concat([df, title_syntax_statis, desc_syntax_statis], axis=1)\n",
    "del desc_syntax_statis,title_syntax_statis\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "corpus length 6947808\n",
      "[('the', 68267), ('a', 42792), ('and', 36092), ('to', 30855), ('of', 30574), ('is', 22159), ('i', 20368), ('it', 20056), ('in', 17853), ('that', 14880), ('this', 14203), ('for', 11865), ('s', 10805), ('with', 10333), ('you', 9359), ('on', 8517), ('but', 7731), ('are', 7675), ('wa', 6990), ('not', 6921)]\n",
      "(200000, 20)\n",
      "corpus length 6656966\n",
      "[('the', 34165), ('a', 32893), ('and', 24361), ('of', 23987), ('for', 20981), ('to', 17728), ('but', 14063), ('it', 10636), ('s', 9687), ('is', 9547), ('great', 9397), ('in', 9141), ('this', 8547), ('good', 8473), ('not', 8289), ('with', 7293), ('i', 6997), ('book', 6953), ('you', 6776), ('on', 5945)]\n",
      "(200000, 20)\n",
      "(200000, 61)\n"
     ]
    }
   ],
   "source": [
    "#bag of words feature\n",
    "def BagOfWords(all_text,chapters,num_of_words):\n",
    "    \"\"\"\n",
    "    Compute the bag of words feature vectors, based on the most common words\n",
    "     in the whole book\n",
    "    \"\"\"\n",
    "    # get most common words in the whole text_file\n",
    "    NUM_TOP_WORDS = num_of_words\n",
    "    tokens = nltk.word_tokenize(all_text)\n",
    "    tokens = [w.lower() for w in tokens]\n",
    "    \n",
    "    # remove punctuation from each word\n",
    "    table = str.maketrans('', '', string.punctuation)\n",
    "    stripped = [w.translate(table) for w in tokens]\n",
    "    #remove remaining tokens that are not alphabetic\n",
    "    words = [word for word in stripped if word.isalpha()]\n",
    "    \n",
    "    #lemmatize\n",
    "    wnl = WordNetLemmatizer() \n",
    "    words = [wnl.lemmatize(w) for w in words]\n",
    "    \n",
    "    fdist = nltk.FreqDist(words)\n",
    "    #print(fdist)\n",
    "    vocab= fdist.most_common(num_of_words)\n",
    "    print(vocab)\n",
    "\n",
    "    # use sklearn to create the bag for words feature vector for each chapter\n",
    "    vectorizer = CountVectorizer(vocabulary=vocab, tokenizer=nltk.word_tokenize)\n",
    "    fvs_bow = vectorizer.fit_transform(chapters).toarray().astype(np.float64)\n",
    "\n",
    "    # normalise by dividing each row by its Euclidean norm\n",
    "    fvs_bow /= 0.00001+np.c_[np.apply_along_axis(np.linalg.norm, 1, fvs_bow)]\n",
    "    return fvs_bow\n",
    "\n",
    "all_text = reduce(lambda x,y: x+y, random.sample(df['description'].values.tolist(),5000))\n",
    "print(\"corpus length\",len(all_text))\n",
    "chapters = df['description'].values.tolist()\n",
    "num_of_words = 20\n",
    "desc_bow = BagOfWords(all_text,chapters, num_of_words)\n",
    "print(desc_bow.shape)\n",
    "desc_bow =  pd.DataFrame(desc_bow)\n",
    "desc_bow.columns = ['desc_bow_'+str(i+1) for i in range(num_of_words)]\n",
    "\n",
    "\n",
    "all_text = reduce(lambda x,y: x+y, df['title'].values.tolist())\n",
    "print(\"corpus length\",len(all_text))\n",
    "chapters = df['title'].values.tolist()\n",
    "num_of_words = 20\n",
    "title_bow = BagOfWords(all_text,chapters, num_of_words)\n",
    "print(title_bow.shape)\n",
    "title_bow =  pd.DataFrame(title_bow)\n",
    "title_bow.columns = ['title_bow_'+str(i+1) for i in range(num_of_words)]\n",
    "\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "df = pd.concat([df, title_bow, desc_bow], axis=1)\n",
    "del desc_bow,title_bow\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n##sentence embedding, 700dim\\nimport sent2vec\\nmodel = sent2vec.Sent2vecModel()\\nmodel.load_model('torontobooks_unigrams.bin')\\nn_dim = 700\\ntitle_emb =  pd.DataFrame(model.embed_sentences(df['title'].values))\\ntitle_emb.columns = ['title_emb_'+str(i+1) for i in range(n_dim)]\\n\\ndesc_emb =  pd.DataFrame(model.embed_sentences(df['description'].values))\\ndesc_emb.columns = ['desc_emb_'+str(i+1) for i in range(n_dim)]\\n#desc_emb.head()\\n\\n## embedding is too large, can't use it\\ndef emb_drop(emb,col_prefix):\\n    for i in range(n_dim):\\n        if i%10 !=0: #drop 9/10\\n            temp = col_prefix+str(i+1)\\n            emb = emb.drop([temp],axis=1)\\n    return emb\\n            \\ntitle_emb=emb_drop(title_emb,'title_emb_')\\ndesc_emb=emb_drop(desc_emb,'desc_emb_')\\nprint(title_emb.shape)\\nprint(desc_emb.shape)\\ndf.reset_index(inplace=True, drop=True)\\ndf = pd.concat([df, title_emb, desc_emb], axis=1)\\ndel desc_emb,title_emb\\nprint(df.shape)\\n\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "##sentence embedding, 700dim\n",
    "import sent2vec\n",
    "model = sent2vec.Sent2vecModel()\n",
    "model.load_model('torontobooks_unigrams.bin')\n",
    "n_dim = 700\n",
    "title_emb =  pd.DataFrame(model.embed_sentences(df['title'].values))\n",
    "title_emb.columns = ['title_emb_'+str(i+1) for i in range(n_dim)]\n",
    "\n",
    "desc_emb =  pd.DataFrame(model.embed_sentences(df['description'].values))\n",
    "desc_emb.columns = ['desc_emb_'+str(i+1) for i in range(n_dim)]\n",
    "#desc_emb.head()\n",
    "\n",
    "## embedding is too large, can't use it\n",
    "def emb_drop(emb,col_prefix):\n",
    "    for i in range(n_dim):\n",
    "        if i%10 !=0: #drop 9/10\n",
    "            temp = col_prefix+str(i+1)\n",
    "            emb = emb.drop([temp],axis=1)\n",
    "    return emb\n",
    "            \n",
    "title_emb=emb_drop(title_emb,'title_emb_')\n",
    "desc_emb=emb_drop(desc_emb,'desc_emb_')\n",
    "print(title_emb.shape)\n",
    "print(desc_emb.shape)\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "df = pd.concat([df, title_emb, desc_emb], axis=1)\n",
    "del desc_emb,title_emb\n",
    "print(df.shape)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>custom_id</th>\n",
       "      <th>title_tag_1</th>\n",
       "      <th>title_tag_2</th>\n",
       "      <th>title_tag_3</th>\n",
       "      <th>title_tag_4</th>\n",
       "      <th>title_tag_5</th>\n",
       "      <th>title_tag_6</th>\n",
       "      <th>title_tag_7</th>\n",
       "      <th>...</th>\n",
       "      <th>description_num_words</th>\n",
       "      <th>title_num_words</th>\n",
       "      <th>description_easy_level</th>\n",
       "      <th>title_easy_level</th>\n",
       "      <th>description_FK_level</th>\n",
       "      <th>title_FK_level</th>\n",
       "      <th>description_GF_scale</th>\n",
       "      <th>title_GF_scale</th>\n",
       "      <th>description_LWF</th>\n",
       "      <th>title_LWF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is nothing special about this streamer. ...</td>\n",
       "      <td>Mediocre</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>335</td>\n",
       "      <td>1</td>\n",
       "      <td>67.18</td>\n",
       "      <td>36.62</td>\n",
       "      <td>9.1</td>\n",
       "      <td>8.4</td>\n",
       "      <td>14.81</td>\n",
       "      <td>40.4</td>\n",
       "      <td>7.285714</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For starters, I did not receive the keyboard t...</td>\n",
       "      <td>Can't Use</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>218</td>\n",
       "      <td>2</td>\n",
       "      <td>72.56</td>\n",
       "      <td>120.21</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-3.1</td>\n",
       "      <td>13.58</td>\n",
       "      <td>0.8</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>My home is located about half way between wher...</td>\n",
       "      <td>So Far, So Good</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>228</td>\n",
       "      <td>4</td>\n",
       "      <td>77.57</td>\n",
       "      <td>118.18</td>\n",
       "      <td>7.2</td>\n",
       "      <td>-2.3</td>\n",
       "      <td>14.79</td>\n",
       "      <td>1.6</td>\n",
       "      <td>8.666667</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In the box is the quite attractive unit, a fil...</td>\n",
       "      <td>Almost</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>185</td>\n",
       "      <td>1</td>\n",
       "      <td>78.08</td>\n",
       "      <td>36.62</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.4</td>\n",
       "      <td>13.89</td>\n",
       "      <td>0.4</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>-0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I have reviewed this previously but it shows a...</td>\n",
       "      <td>Repeat Review</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>56</td>\n",
       "      <td>2</td>\n",
       "      <td>85.49</td>\n",
       "      <td>35.61</td>\n",
       "      <td>4.1</td>\n",
       "      <td>8.8</td>\n",
       "      <td>10.19</td>\n",
       "      <td>0.8</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 73 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         description            title  \\\n",
       "0  There is nothing special about this streamer. ...         Mediocre   \n",
       "1  For starters, I did not receive the keyboard t...        Can't Use   \n",
       "2  My home is located about half way between wher...  So Far, So Good   \n",
       "3  In the box is the quite attractive unit, a fil...           Almost   \n",
       "4  I have reviewed this previously but it shows a...    Repeat Review   \n",
       "\n",
       "   custom_id  title_tag_1  title_tag_2  title_tag_3  title_tag_4  title_tag_5  \\\n",
       "0          0          1.0     0.000000          0.0          0.0          0.0   \n",
       "1          0          0.0     0.333333          0.0          0.0          0.0   \n",
       "2          0          0.0     0.200000          0.0          0.0          0.2   \n",
       "3          0          0.0     0.000000          0.0          0.0          0.0   \n",
       "4          0          0.0     1.000000          0.0          0.0          0.0   \n",
       "\n",
       "   title_tag_6  title_tag_7    ...      description_num_words  \\\n",
       "0          0.0          0.0    ...                        335   \n",
       "1          0.0          0.0    ...                        218   \n",
       "2          0.0          0.0    ...                        228   \n",
       "3          0.0          0.0    ...                        185   \n",
       "4          0.0          0.0    ...                         56   \n",
       "\n",
       "   title_num_words  description_easy_level  title_easy_level  \\\n",
       "0                1                   67.18             36.62   \n",
       "1                2                   72.56            120.21   \n",
       "2                4                   77.57            118.18   \n",
       "3                1                   78.08             36.62   \n",
       "4                2                   85.49             35.61   \n",
       "\n",
       "   description_FK_level  title_FK_level  description_GF_scale  title_GF_scale  \\\n",
       "0                   9.1             8.4                 14.81            40.4   \n",
       "1                   7.0            -3.1                 13.58             0.8   \n",
       "2                   7.2            -2.3                 14.79             1.6   \n",
       "3                   7.0             8.4                 13.89             0.4   \n",
       "4                   4.1             8.8                 10.19             0.8   \n",
       "\n",
       "   description_LWF  title_LWF  \n",
       "0         7.285714       -0.5  \n",
       "1         6.000000        0.0  \n",
       "2         8.666667        1.0  \n",
       "3         7.000000       -0.5  \n",
       "4         4.000000        0.0  \n",
       "\n",
       "[5 rows x 73 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#feature of readability\n",
    "df['description_syllable_cnt'] = df['description'].apply(lambda x: textstat.syllable_count(x, lang='en_US'))\n",
    "df['title_syllable_cnt'] = df['title'].apply(lambda x: textstat.syllable_count(x, lang='en_US'))\n",
    "\n",
    "#count number of words by removing punction\n",
    "df['description_num_words'] = df['description'].apply(lambda x: textstat.lexicon_count(x, removepunct=True))\n",
    "df['title_num_words'] = df['title'].apply(lambda x: textstat.lexicon_count(x, removepunct=True))\n",
    "\n",
    "df['description_easy_level'] = df['description'].apply(lambda x: textstat.flesch_reading_ease(x))\n",
    "df['title_easy_level'] = df['title'].apply(lambda x: textstat.flesch_reading_ease(x))\n",
    "\n",
    "df['description_FK_level'] = df['description'].apply(lambda x: textstat.flesch_kincaid_grade(x))\n",
    "df['title_FK_level'] = df['title'].apply(lambda x: textstat.flesch_kincaid_grade(x))\n",
    "\n",
    "df['description_GF_scale'] = df['description'].apply(lambda x: textstat.gunning_fog(x))\n",
    "df['title_GF_scale'] = df['title'].apply(lambda x: textstat.gunning_fog(x))\n",
    "\n",
    "df['description_LWF'] = df['description'].apply(lambda x: textstat.linsear_write_formula(x))\n",
    "df['title_LWF'] = df['title'].apply(lambda x: textstat.linsear_write_formula(x))\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>custom_id</th>\n",
       "      <th>title_tag_1</th>\n",
       "      <th>title_tag_2</th>\n",
       "      <th>title_tag_3</th>\n",
       "      <th>title_tag_4</th>\n",
       "      <th>title_tag_5</th>\n",
       "      <th>title_tag_6</th>\n",
       "      <th>title_tag_7</th>\n",
       "      <th>...</th>\n",
       "      <th>description_FK_level</th>\n",
       "      <th>title_FK_level</th>\n",
       "      <th>description_GF_scale</th>\n",
       "      <th>title_GF_scale</th>\n",
       "      <th>description_LWF</th>\n",
       "      <th>title_LWF</th>\n",
       "      <th>description_stp_cnt</th>\n",
       "      <th>title_stp_cnt</th>\n",
       "      <th>description_stp_ratio</th>\n",
       "      <th>title_stp_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is nothing special about this streamer. ...</td>\n",
       "      <td>Mediocre</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.1</td>\n",
       "      <td>8.4</td>\n",
       "      <td>14.81</td>\n",
       "      <td>40.4</td>\n",
       "      <td>7.285714</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>193</td>\n",
       "      <td>0</td>\n",
       "      <td>0.576119</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For starters, I did not receive the keyboard t...</td>\n",
       "      <td>Can't Use</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-3.1</td>\n",
       "      <td>13.58</td>\n",
       "      <td>0.8</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>111</td>\n",
       "      <td>0</td>\n",
       "      <td>0.509174</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>My home is located about half way between wher...</td>\n",
       "      <td>So Far, So Good</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.2</td>\n",
       "      <td>-2.3</td>\n",
       "      <td>14.79</td>\n",
       "      <td>1.6</td>\n",
       "      <td>8.666667</td>\n",
       "      <td>1.0</td>\n",
       "      <td>119</td>\n",
       "      <td>2</td>\n",
       "      <td>0.521930</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In the box is the quite attractive unit, a fil...</td>\n",
       "      <td>Almost</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.4</td>\n",
       "      <td>13.89</td>\n",
       "      <td>0.4</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>89</td>\n",
       "      <td>0</td>\n",
       "      <td>0.481081</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I have reviewed this previously but it shows a...</td>\n",
       "      <td>Repeat Review</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.1</td>\n",
       "      <td>8.8</td>\n",
       "      <td>10.19</td>\n",
       "      <td>0.8</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0.535714</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 77 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         description            title  \\\n",
       "0  There is nothing special about this streamer. ...         Mediocre   \n",
       "1  For starters, I did not receive the keyboard t...        Can't Use   \n",
       "2  My home is located about half way between wher...  So Far, So Good   \n",
       "3  In the box is the quite attractive unit, a fil...           Almost   \n",
       "4  I have reviewed this previously but it shows a...    Repeat Review   \n",
       "\n",
       "   custom_id  title_tag_1  title_tag_2  title_tag_3  title_tag_4  title_tag_5  \\\n",
       "0          0          1.0     0.000000          0.0          0.0          0.0   \n",
       "1          0          0.0     0.333333          0.0          0.0          0.0   \n",
       "2          0          0.0     0.200000          0.0          0.0          0.2   \n",
       "3          0          0.0     0.000000          0.0          0.0          0.0   \n",
       "4          0          0.0     1.000000          0.0          0.0          0.0   \n",
       "\n",
       "   title_tag_6  title_tag_7       ...         description_FK_level  \\\n",
       "0          0.0          0.0       ...                          9.1   \n",
       "1          0.0          0.0       ...                          7.0   \n",
       "2          0.0          0.0       ...                          7.2   \n",
       "3          0.0          0.0       ...                          7.0   \n",
       "4          0.0          0.0       ...                          4.1   \n",
       "\n",
       "   title_FK_level  description_GF_scale  title_GF_scale  description_LWF  \\\n",
       "0             8.4                 14.81            40.4         7.285714   \n",
       "1            -3.1                 13.58             0.8         6.000000   \n",
       "2            -2.3                 14.79             1.6         8.666667   \n",
       "3             8.4                 13.89             0.4         7.000000   \n",
       "4             8.8                 10.19             0.8         4.000000   \n",
       "\n",
       "   title_LWF  description_stp_cnt  title_stp_cnt  description_stp_ratio  \\\n",
       "0       -0.5                  193              0               0.576119   \n",
       "1        0.0                  111              0               0.509174   \n",
       "2        1.0                  119              2               0.521930   \n",
       "3       -0.5                   89              0               0.481081   \n",
       "4        0.0                   30              0               0.535714   \n",
       "\n",
       "   title_stp_ratio  \n",
       "0              0.0  \n",
       "1              0.0  \n",
       "2              0.5  \n",
       "3              0.0  \n",
       "4              0.0  \n",
       "\n",
       "[5 rows x 77 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#feature from stop words\n",
    "def stop_word_cnt(text):\n",
    "    # split into words\n",
    "    tokens = word_tokenize(text)\n",
    "    # convert to lower case\n",
    "    tokens = [w.lower() for w in tokens]\n",
    "    \n",
    "    # remove punctuation from each word\n",
    "    table = str.maketrans('', '', string.punctuation)\n",
    "    stripped = [w.translate(table) for w in tokens]\n",
    "    #remove remaining tokens that are not alphabetic\n",
    "    words = [word for word in stripped if word.isalpha()]\n",
    "    \n",
    "    # filter out stop words, and do lemmatizing\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    stops = [w for w in words if w in stop_words]\n",
    "    return len(stops)\n",
    "\n",
    "df['description_stp_cnt'] = df['description'].apply(lambda x: stop_word_cnt(x))\n",
    "df['title_stp_cnt'] = df['title'].apply(lambda x: stop_word_cnt(x))\n",
    "df['description_stp_ratio'] = df['description_stp_cnt']/df['description_num_words']\n",
    "df['title_stp_ratio'] = df['title_stp_cnt']/df['title_num_words']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 117)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_cols= ['description','title']\n",
    "for cols in text_cols:\n",
    "    df[cols] = df[cols].astype(str) \n",
    "    df[cols] = df[cols].astype(str).fillna('missing') # FILL NA\n",
    "    df[cols + '_len'] = df[cols].map(lambda x: len(str(x))).astype(np.float16) #Lenth\n",
    "    df[cols + \"_upper_ratio\"] = df[cols].apply(lambda x: sum(map(str.isupper, x)))/df[cols + '_len']\n",
    "    df[cols + \"_space_ratio\"] = df[cols].apply(lambda x: x.count(' '))/df[cols + '_len']\n",
    "    #df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n",
    "    #df[cols + '_num_words'] = df[cols].apply(lambda comment: len(word_tokenize(comment)) # Count number of Words\n",
    "    df[cols] = df[cols].str.lower() # Lowercase all text, so that capitalized words dont get treated differently\n",
    "    df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "    df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] # Count Unique Words\n",
    "    #df[cols + '_num_letters'] = df[cols].apply(lambda comment: len(comment)) # Count number of Letters\n",
    "\n",
    "\n",
    "# Extra Feature Engineering\n",
    "df['title_desc_len_ratio'] = df['title_num_words']/df['description_num_words']\n",
    "df['description_sentence_num'] = df['description'].apply(lambda comment: len(sent_tokenize(comment)))   \n",
    "df['title_sentence_num'] = df['title'].apply(lambda comment: len(sent_tokenize(comment)))\n",
    "df['title_desc_sentence_ratio'] =  df['title_sentence_num']/df['description_sentence_num']  \n",
    "df['average_desc_sentence_words'] = df['description_num_words']/df['description_sentence_num']\n",
    "df['average_desc_sentence_len'] = df['description_len']/df['description_sentence_num']\n",
    "df['average_title_sentence_words'] = df['title_num_words']/df['title_sentence_num']\n",
    "df['average_title_sentence_len'] = df['title_len']/df['title_sentence_num']\n",
    "\n",
    "df['description_average_words_len'] = df['description_len']/df['description_num_words'] \n",
    "df['title_average_words_len'] = df['title_len']/df['title_num_words']\n",
    "\n",
    "#punction features\n",
    "df['description_comma_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count(','))/df['description_sentence_num']\n",
    "df['description_semicolon_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count(';'))/df['description_sentence_num']\n",
    "df['description_colon_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count(':'))/df['description_sentence_num']\n",
    "df['description_excla_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count('!'))/df['description_sentence_num']\n",
    "df['description_brack_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count('('))/df['description_sentence_num']\n",
    "df['description_ques_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count('?'))/df['description_sentence_num']\n",
    "df['description_dot_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count('.'))/df['description_sentence_num']\n",
    "df['description_hyphen_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count('-'))/df['description_sentence_num']\n",
    "df['description_quat_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count('\"'))/df['description_sentence_num']\n",
    "df['description_and_ratio'] = df['description'].apply(lambda x: word_tokenize(x).count('&'))/df['description_sentence_num']\n",
    "\n",
    "df['title_comma_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count(','))/df['title_sentence_num']\n",
    "df['title_semicolon_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count(';'))/df['title_sentence_num']\n",
    "df['title_colon_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count(':'))/df['title_sentence_num']\n",
    "df['title_excla_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count('!'))/df['title_sentence_num']\n",
    "df['title_brack_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count('('))/df['title_sentence_num']\n",
    "df['title_ques_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count('?'))/df['title_sentence_num']\n",
    "df['title_dot_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count('.'))/df['title_sentence_num']\n",
    "df['title_hypen_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count('-'))/df['title_sentence_num']\n",
    "df['title_quat_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count('\"'))/df['title_sentence_num']\n",
    "df['title_and_ratio'] = df['title'].apply(lambda x: word_tokenize(x).count('&'))/df['title_sentence_num']\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>svd_title_1</th>\n",
       "      <th>svd_title_2</th>\n",
       "      <th>svd_title_3</th>\n",
       "      <th>svd_title_4</th>\n",
       "      <th>svd_title_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.002360</td>\n",
       "      <td>-0.000294</td>\n",
       "      <td>0.000275</td>\n",
       "      <td>-0.000680</td>\n",
       "      <td>0.000186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.040390</td>\n",
       "      <td>-0.021127</td>\n",
       "      <td>-0.004192</td>\n",
       "      <td>0.043002</td>\n",
       "      <td>0.015997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.122196</td>\n",
       "      <td>-0.058713</td>\n",
       "      <td>0.238371</td>\n",
       "      <td>-0.068152</td>\n",
       "      <td>-0.027490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.006967</td>\n",
       "      <td>-0.001193</td>\n",
       "      <td>0.001091</td>\n",
       "      <td>0.003315</td>\n",
       "      <td>-0.000493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.009681</td>\n",
       "      <td>-0.006186</td>\n",
       "      <td>-0.005841</td>\n",
       "      <td>-0.003848</td>\n",
       "      <td>0.006604</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   svd_title_1  svd_title_2  svd_title_3  svd_title_4  svd_title_5\n",
       "0     0.002360    -0.000294     0.000275    -0.000680     0.000186\n",
       "1     0.040390    -0.021127    -0.004192     0.043002     0.015997\n",
       "2     0.122196    -0.058713     0.238371    -0.068152    -0.027490\n",
       "3     0.006967    -0.001193     0.001091     0.003315    -0.000493\n",
       "4     0.009681    -0.006186    -0.005841    -0.003848     0.006604"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### TFIDF Vectorizer for title###\n",
    "tfidf_vec = TfidfVectorizer(ngram_range=(1,1))\n",
    "title_tfidf = tfidf_vec.fit_transform(df['title'].values.tolist() )\n",
    "#print(title_tfidf)\n",
    "### SVD Components ###\n",
    "n_comp = 5\n",
    "svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\n",
    "svd_obj.fit(title_tfidf)\n",
    "title_svd = pd.DataFrame(svd_obj.transform(title_tfidf))\n",
    "title_svd.columns = ['svd_title_'+str(i+1) for i in range(n_comp)]\n",
    "title_svd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>svd_desc_1</th>\n",
       "      <th>svd_desc_2</th>\n",
       "      <th>svd_desc_3</th>\n",
       "      <th>svd_desc_4</th>\n",
       "      <th>svd_desc_5</th>\n",
       "      <th>svd_desc_6</th>\n",
       "      <th>svd_desc_7</th>\n",
       "      <th>svd_desc_8</th>\n",
       "      <th>svd_desc_9</th>\n",
       "      <th>svd_desc_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.147411</td>\n",
       "      <td>0.028393</td>\n",
       "      <td>-0.008297</td>\n",
       "      <td>-0.012855</td>\n",
       "      <td>-0.028956</td>\n",
       "      <td>-0.069639</td>\n",
       "      <td>-0.007347</td>\n",
       "      <td>0.031262</td>\n",
       "      <td>-0.006619</td>\n",
       "      <td>0.013463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.127627</td>\n",
       "      <td>0.096277</td>\n",
       "      <td>0.006690</td>\n",
       "      <td>-0.034563</td>\n",
       "      <td>0.013155</td>\n",
       "      <td>-0.070372</td>\n",
       "      <td>0.031915</td>\n",
       "      <td>-0.003273</td>\n",
       "      <td>0.004753</td>\n",
       "      <td>-0.031798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.103958</td>\n",
       "      <td>0.064329</td>\n",
       "      <td>-0.024244</td>\n",
       "      <td>-0.037800</td>\n",
       "      <td>-0.012409</td>\n",
       "      <td>-0.019035</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.020755</td>\n",
       "      <td>0.012235</td>\n",
       "      <td>-0.015715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.116759</td>\n",
       "      <td>0.104831</td>\n",
       "      <td>-0.035261</td>\n",
       "      <td>-0.060978</td>\n",
       "      <td>-0.006680</td>\n",
       "      <td>-0.028974</td>\n",
       "      <td>0.007375</td>\n",
       "      <td>-0.022876</td>\n",
       "      <td>-0.014260</td>\n",
       "      <td>-0.041198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.046275</td>\n",
       "      <td>0.028150</td>\n",
       "      <td>-0.010482</td>\n",
       "      <td>-0.011946</td>\n",
       "      <td>-0.008308</td>\n",
       "      <td>0.001104</td>\n",
       "      <td>-0.038105</td>\n",
       "      <td>-0.021703</td>\n",
       "      <td>-0.002910</td>\n",
       "      <td>0.004174</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   svd_desc_1  svd_desc_2  svd_desc_3  svd_desc_4  svd_desc_5  svd_desc_6  \\\n",
       "0    0.147411    0.028393   -0.008297   -0.012855   -0.028956   -0.069639   \n",
       "1    0.127627    0.096277    0.006690   -0.034563    0.013155   -0.070372   \n",
       "2    0.103958    0.064329   -0.024244   -0.037800   -0.012409   -0.019035   \n",
       "3    0.116759    0.104831   -0.035261   -0.060978   -0.006680   -0.028974   \n",
       "4    0.046275    0.028150   -0.010482   -0.011946   -0.008308    0.001104   \n",
       "\n",
       "   svd_desc_7  svd_desc_8  svd_desc_9  svd_desc_10  \n",
       "0   -0.007347    0.031262   -0.006619     0.013463  \n",
       "1    0.031915   -0.003273    0.004753    -0.031798  \n",
       "2   -0.000008   -0.020755    0.012235    -0.015715  \n",
       "3    0.007375   -0.022876   -0.014260    -0.041198  \n",
       "4   -0.038105   -0.021703   -0.002910     0.004174  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### TFIDF Vectorizer for description###\n",
    "#get from https://www.kaggle.com/samratp/avito-lightgbm-with-ridge-feature-v-3-0-0-2219/code\n",
    "english_stop = set(stopwords.words('english'))\n",
    "tfidf_para = {\n",
    "    \"stop_words\": english_stop,\n",
    "    \"analyzer\": 'word',\n",
    "    \"token_pattern\": r'\\w{1,}',\n",
    "    \"sublinear_tf\": True,\n",
    "    \"dtype\": np.float32,\n",
    "    \"norm\": 'l2',\n",
    "    #\"min_df\":5,\n",
    "    #\"max_df\":.9,\n",
    "    \"smooth_idf\":False\n",
    "}\n",
    "\n",
    "\n",
    "tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=20000,**tfidf_para)  ##** means as dictionary\n",
    "des_tfidf = tfidf_vec.fit_transform(df['description'].values.tolist() )\n",
    "\n",
    "### SVD Components ###\n",
    "n_comp = 10\n",
    "svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\n",
    "svd_obj.fit(des_tfidf)\n",
    "des_svd = pd.DataFrame(svd_obj.transform(des_tfidf))\n",
    "des_svd.columns = ['svd_desc_'+str(i+1) for i in range(n_comp)]\n",
    "des_svd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(inplace=True, drop=True)\n",
    "df = pd.concat([df, title_svd, des_svd], axis=1)\n",
    "del title_tfidf,des_tfidf, title_svd, des_svd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfrom sklearn.feature_extraction.text import HashingVectorizer\\nnum_desc_features = 20\\nnum_title_features = 5\\ndef desc_vectorizer(text):\\n    hv = HashingVectorizer(n_features=num_desc_features)\\n    encode = hv.transform(text).toarray()\\n    return encode\\n\\ndef title_vectorizer(text):\\n    hv = HashingVectorizer(n_features=num_title_features)\\n    encode = hv.transform(text).toarray()\\n    return encode\\n\\ndf['description'] =  df['description'].apply(lambda x: desc_vectorizer([x])) \\ndf['title'] = df['title'].apply(lambda x: title_vectorizer([x])) \\n\\ndf.reset_index(inplace=True, drop=True)\\nfor i in range(num_desc_features):\\n    temp_desc = []\\n    for j in range(50000):\\n        temp = df['description'].values\\n        temp_desc.append(temp[j][0][i])\\n    name = 'desc_hash_'+str(i)\\n    desc_frame = pd.DataFrame({name:temp_desc})\\n    df = pd.concat([df,desc_frame],axis=1)\\nfor i in range(num_title_features):\\n    temp_title = []\\n    for j in range(50000):\\n        temp = df['title'].values\\n        temp_title.append(temp[j][0][i])\\n    name = 'title_hash_'+str(i)\\n    title_frame = pd.DataFrame({name:temp_title}) \\n    df = pd.concat([df,title_frame],axis=1)\\n    \\ndf.head()\\n\""
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "num_desc_features = 20\n",
    "num_title_features = 5\n",
    "def desc_vectorizer(text):\n",
    "    hv = HashingVectorizer(n_features=num_desc_features)\n",
    "    encode = hv.transform(text).toarray()\n",
    "    return encode\n",
    "\n",
    "def title_vectorizer(text):\n",
    "    hv = HashingVectorizer(n_features=num_title_features)\n",
    "    encode = hv.transform(text).toarray()\n",
    "    return encode\n",
    "\n",
    "df['description'] =  df['description'].apply(lambda x: desc_vectorizer([x])) \n",
    "df['title'] = df['title'].apply(lambda x: title_vectorizer([x])) \n",
    "\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "for i in range(num_desc_features):\n",
    "    temp_desc = []\n",
    "    for j in range(50000):\n",
    "        temp = df['description'].values\n",
    "        temp_desc.append(temp[j][0][i])\n",
    "    name = 'desc_hash_'+str(i)\n",
    "    desc_frame = pd.DataFrame({name:temp_desc})\n",
    "    df = pd.concat([df,desc_frame],axis=1)\n",
    "for i in range(num_title_features):\n",
    "    temp_title = []\n",
    "    for j in range(50000):\n",
    "        temp = df['title'].values\n",
    "        temp_title.append(temp[j][0][i])\n",
    "    name = 'title_hash_'+str(i)\n",
    "    title_frame = pd.DataFrame({name:temp_title}) \n",
    "    df = pd.concat([df,title_frame],axis=1)\n",
    "    \n",
    "df.head()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(200000, 130)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>custom_id</th>\n",
       "      <th>title_tag_1</th>\n",
       "      <th>title_tag_2</th>\n",
       "      <th>title_tag_3</th>\n",
       "      <th>title_tag_4</th>\n",
       "      <th>title_tag_5</th>\n",
       "      <th>title_tag_6</th>\n",
       "      <th>title_tag_7</th>\n",
       "      <th>title_tag_8</th>\n",
       "      <th>title_tag_9</th>\n",
       "      <th>...</th>\n",
       "      <th>svd_desc_1</th>\n",
       "      <th>svd_desc_2</th>\n",
       "      <th>svd_desc_3</th>\n",
       "      <th>svd_desc_4</th>\n",
       "      <th>svd_desc_5</th>\n",
       "      <th>svd_desc_6</th>\n",
       "      <th>svd_desc_7</th>\n",
       "      <th>svd_desc_8</th>\n",
       "      <th>svd_desc_9</th>\n",
       "      <th>svd_desc_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.147411</td>\n",
       "      <td>0.028393</td>\n",
       "      <td>-0.008297</td>\n",
       "      <td>-0.012855</td>\n",
       "      <td>-0.028956</td>\n",
       "      <td>-0.069639</td>\n",
       "      <td>-0.007347</td>\n",
       "      <td>0.031262</td>\n",
       "      <td>-0.006619</td>\n",
       "      <td>0.013463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.127627</td>\n",
       "      <td>0.096277</td>\n",
       "      <td>0.006690</td>\n",
       "      <td>-0.034563</td>\n",
       "      <td>0.013155</td>\n",
       "      <td>-0.070372</td>\n",
       "      <td>0.031915</td>\n",
       "      <td>-0.003273</td>\n",
       "      <td>0.004753</td>\n",
       "      <td>-0.031798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.103958</td>\n",
       "      <td>0.064329</td>\n",
       "      <td>-0.024244</td>\n",
       "      <td>-0.037800</td>\n",
       "      <td>-0.012409</td>\n",
       "      <td>-0.019035</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.020755</td>\n",
       "      <td>0.012235</td>\n",
       "      <td>-0.015715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.116759</td>\n",
       "      <td>0.104831</td>\n",
       "      <td>-0.035261</td>\n",
       "      <td>-0.060978</td>\n",
       "      <td>-0.006680</td>\n",
       "      <td>-0.028974</td>\n",
       "      <td>0.007375</td>\n",
       "      <td>-0.022876</td>\n",
       "      <td>-0.014260</td>\n",
       "      <td>-0.041198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.046275</td>\n",
       "      <td>0.028150</td>\n",
       "      <td>-0.010482</td>\n",
       "      <td>-0.011946</td>\n",
       "      <td>-0.008308</td>\n",
       "      <td>0.001104</td>\n",
       "      <td>-0.038105</td>\n",
       "      <td>-0.021703</td>\n",
       "      <td>-0.002910</td>\n",
       "      <td>0.004174</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 130 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   custom_id  title_tag_1  title_tag_2  title_tag_3  title_tag_4  title_tag_5  \\\n",
       "0          0          1.0     0.000000          0.0          0.0          0.0   \n",
       "1          0          0.0     0.333333          0.0          0.0          0.0   \n",
       "2          0          0.0     0.200000          0.0          0.0          0.2   \n",
       "3          0          0.0     0.000000          0.0          0.0          0.0   \n",
       "4          0          0.0     1.000000          0.0          0.0          0.0   \n",
       "\n",
       "   title_tag_6  title_tag_7  title_tag_8  title_tag_9     ...       \\\n",
       "0          0.0          0.0          0.0          0.0     ...        \n",
       "1          0.0          0.0          0.0          0.0     ...        \n",
       "2          0.0          0.0          0.0          0.0     ...        \n",
       "3          0.0          0.0          0.0          0.0     ...        \n",
       "4          0.0          0.0          0.0          0.0     ...        \n",
       "\n",
       "   svd_desc_1  svd_desc_2  svd_desc_3  svd_desc_4  svd_desc_5  svd_desc_6  \\\n",
       "0    0.147411    0.028393   -0.008297   -0.012855   -0.028956   -0.069639   \n",
       "1    0.127627    0.096277    0.006690   -0.034563    0.013155   -0.070372   \n",
       "2    0.103958    0.064329   -0.024244   -0.037800   -0.012409   -0.019035   \n",
       "3    0.116759    0.104831   -0.035261   -0.060978   -0.006680   -0.028974   \n",
       "4    0.046275    0.028150   -0.010482   -0.011946   -0.008308    0.001104   \n",
       "\n",
       "   svd_desc_7  svd_desc_8  svd_desc_9  svd_desc_10  \n",
       "0   -0.007347    0.031262   -0.006619     0.013463  \n",
       "1    0.031915   -0.003273    0.004753    -0.031798  \n",
       "2   -0.000008   -0.020755    0.012235    -0.015715  \n",
       "3    0.007375   -0.022876   -0.014260    -0.041198  \n",
       "4   -0.038105   -0.021703   -0.002910     0.004174  \n",
       "\n",
       "[5 rows x 130 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop(['description','title'],axis=1)\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 188.45 MB\n",
      "Memory usage after optimization is: 50.93 MB\n",
      "Decreased by 73.0%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>custom_id</th>\n",
       "      <th>title_tag_1</th>\n",
       "      <th>title_tag_2</th>\n",
       "      <th>title_tag_3</th>\n",
       "      <th>title_tag_4</th>\n",
       "      <th>title_tag_5</th>\n",
       "      <th>title_tag_6</th>\n",
       "      <th>title_tag_7</th>\n",
       "      <th>title_tag_8</th>\n",
       "      <th>title_tag_9</th>\n",
       "      <th>...</th>\n",
       "      <th>svd_desc_1</th>\n",
       "      <th>svd_desc_2</th>\n",
       "      <th>svd_desc_3</th>\n",
       "      <th>svd_desc_4</th>\n",
       "      <th>svd_desc_5</th>\n",
       "      <th>svd_desc_6</th>\n",
       "      <th>svd_desc_7</th>\n",
       "      <th>svd_desc_8</th>\n",
       "      <th>svd_desc_9</th>\n",
       "      <th>svd_desc_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.147461</td>\n",
       "      <td>0.028397</td>\n",
       "      <td>-0.008293</td>\n",
       "      <td>-0.012856</td>\n",
       "      <td>-0.028961</td>\n",
       "      <td>-0.069641</td>\n",
       "      <td>-0.007347</td>\n",
       "      <td>0.031250</td>\n",
       "      <td>-0.006618</td>\n",
       "      <td>0.013466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333252</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.127686</td>\n",
       "      <td>0.096252</td>\n",
       "      <td>0.006691</td>\n",
       "      <td>-0.034576</td>\n",
       "      <td>0.013153</td>\n",
       "      <td>-0.070374</td>\n",
       "      <td>0.031921</td>\n",
       "      <td>-0.003273</td>\n",
       "      <td>0.004753</td>\n",
       "      <td>-0.031799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.199951</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.199951</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.103943</td>\n",
       "      <td>0.064331</td>\n",
       "      <td>-0.024246</td>\n",
       "      <td>-0.037811</td>\n",
       "      <td>-0.012405</td>\n",
       "      <td>-0.019043</td>\n",
       "      <td>-0.000008</td>\n",
       "      <td>-0.020752</td>\n",
       "      <td>0.012238</td>\n",
       "      <td>-0.015717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.116760</td>\n",
       "      <td>0.104858</td>\n",
       "      <td>-0.035248</td>\n",
       "      <td>-0.060974</td>\n",
       "      <td>-0.006680</td>\n",
       "      <td>-0.028976</td>\n",
       "      <td>0.007374</td>\n",
       "      <td>-0.022873</td>\n",
       "      <td>-0.014259</td>\n",
       "      <td>-0.041199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.046265</td>\n",
       "      <td>0.028152</td>\n",
       "      <td>-0.010483</td>\n",
       "      <td>-0.011948</td>\n",
       "      <td>-0.008308</td>\n",
       "      <td>0.001104</td>\n",
       "      <td>-0.038116</td>\n",
       "      <td>-0.021698</td>\n",
       "      <td>-0.002911</td>\n",
       "      <td>0.004173</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 130 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   custom_id  title_tag_1  title_tag_2  title_tag_3  title_tag_4  title_tag_5  \\\n",
       "0          0          1.0     0.000000          0.0          0.0     0.000000   \n",
       "1          0          0.0     0.333252          0.0          0.0     0.000000   \n",
       "2          0          0.0     0.199951          0.0          0.0     0.199951   \n",
       "3          0          0.0     0.000000          0.0          0.0     0.000000   \n",
       "4          0          0.0     1.000000          0.0          0.0     0.000000   \n",
       "\n",
       "   title_tag_6  title_tag_7  title_tag_8  title_tag_9     ...       \\\n",
       "0          0.0          0.0          0.0          0.0     ...        \n",
       "1          0.0          0.0          0.0          0.0     ...        \n",
       "2          0.0          0.0          0.0          0.0     ...        \n",
       "3          0.0          0.0          0.0          0.0     ...        \n",
       "4          0.0          0.0          0.0          0.0     ...        \n",
       "\n",
       "   svd_desc_1  svd_desc_2  svd_desc_3  svd_desc_4  svd_desc_5  svd_desc_6  \\\n",
       "0    0.147461    0.028397   -0.008293   -0.012856   -0.028961   -0.069641   \n",
       "1    0.127686    0.096252    0.006691   -0.034576    0.013153   -0.070374   \n",
       "2    0.103943    0.064331   -0.024246   -0.037811   -0.012405   -0.019043   \n",
       "3    0.116760    0.104858   -0.035248   -0.060974   -0.006680   -0.028976   \n",
       "4    0.046265    0.028152   -0.010483   -0.011948   -0.008308    0.001104   \n",
       "\n",
       "   svd_desc_7  svd_desc_8  svd_desc_9  svd_desc_10  \n",
       "0   -0.007347    0.031250   -0.006618     0.013466  \n",
       "1    0.031921   -0.003273    0.004753    -0.031799  \n",
       "2   -0.000008   -0.020752    0.012238    -0.015717  \n",
       "3    0.007374   -0.022873   -0.014259    -0.041199  \n",
       "4   -0.038116   -0.021698   -0.002911     0.004173  \n",
       "\n",
       "[5 rows x 130 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Thanks You Guillaume Martin for the Awesome Memory Optimizer!\n",
    "# https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else: df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "df = reduce_mem_usage(df)    #reduce memory\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#write df to file\n",
    "df.to_csv(\"my_all_data.csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}