letter_frequency_lang_detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from letter_frequency_lang_detection_utils import *"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading the characters frequency data by language"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Character frequency by language table available in: [Wikipedia](https://en.wikipedia.org/wiki/Letter_frequency)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(87, 17)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "characters_freq_path = './languages_letter_frequencies/' + \\\n",
    "                       'languages_letter_frequencies.html'\n",
    "\n",
    "df_freq_raw = pd.read_html(characters_freq_path, encoding='utf-8')[0]\n",
    "df_freq_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>english</th>\n",
       "      <th>french</th>\n",
       "      <th>german</th>\n",
       "      <th>spanish</th>\n",
       "      <th>portuguese</th>\n",
       "      <th>esperanto</th>\n",
       "      <th>italian</th>\n",
       "      <th>turkish</th>\n",
       "      <th>swedish</th>\n",
       "      <th>polish</th>\n",
       "      <th>dutch</th>\n",
       "      <th>danish</th>\n",
       "      <th>icelandic</th>\n",
       "      <th>finnish</th>\n",
       "      <th>czech</th>\n",
       "      <th>hungarian</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>letter</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>a</th>\n",
       "      <td>8.167%</td>\n",
       "      <td>7.636%</td>\n",
       "      <td>6.516%</td>\n",
       "      <td>11.525%</td>\n",
       "      <td>14.634%</td>\n",
       "      <td>12.117%</td>\n",
       "      <td>11.745%</td>\n",
       "      <td>11.920%</td>\n",
       "      <td>9.383%</td>\n",
       "      <td>8.965%</td>\n",
       "      <td>7.49%</td>\n",
       "      <td>6.025%</td>\n",
       "      <td>10.110%</td>\n",
       "      <td>12.217%</td>\n",
       "      <td>8.421%</td>\n",
       "      <td>10.778%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b</th>\n",
       "      <td>1.492%</td>\n",
       "      <td>0.901%</td>\n",
       "      <td>1.886%</td>\n",
       "      <td>2.215%</td>\n",
       "      <td>1.043%</td>\n",
       "      <td>0.980%</td>\n",
       "      <td>0.927%</td>\n",
       "      <td>2.844%</td>\n",
       "      <td>1.535%</td>\n",
       "      <td>1.482%</td>\n",
       "      <td>1.58%</td>\n",
       "      <td>2.000%</td>\n",
       "      <td>1.043%</td>\n",
       "      <td>0.281%</td>\n",
       "      <td>0.822%</td>\n",
       "      <td>2.647%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c</th>\n",
       "      <td>2.782%</td>\n",
       "      <td>3.260%</td>\n",
       "      <td>2.732%</td>\n",
       "      <td>4.019%</td>\n",
       "      <td>3.882%</td>\n",
       "      <td>0.776%</td>\n",
       "      <td>4.501%</td>\n",
       "      <td>0.963%</td>\n",
       "      <td>1.486%</td>\n",
       "      <td>3.988%</td>\n",
       "      <td>1.24%</td>\n",
       "      <td>0.565%</td>\n",
       "      <td>~0%</td>\n",
       "      <td>0.281%</td>\n",
       "      <td>0.740%</td>\n",
       "      <td>0.924%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d</th>\n",
       "      <td>4.253%</td>\n",
       "      <td>3.669%</td>\n",
       "      <td>5.076%</td>\n",
       "      <td>5.010%</td>\n",
       "      <td>4.992%</td>\n",
       "      <td>3.044%</td>\n",
       "      <td>3.736%</td>\n",
       "      <td>4.706%</td>\n",
       "      <td>4.702%</td>\n",
       "      <td>3.293%</td>\n",
       "      <td>5.93%</td>\n",
       "      <td>5.858%</td>\n",
       "      <td>1.575%</td>\n",
       "      <td>1.043%</td>\n",
       "      <td>3.475%</td>\n",
       "      <td>2.410%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e</th>\n",
       "      <td>12.702%</td>\n",
       "      <td>14.715%</td>\n",
       "      <td>16.396%</td>\n",
       "      <td>12.181%</td>\n",
       "      <td>12.570%</td>\n",
       "      <td>8.995%</td>\n",
       "      <td>11.792%</td>\n",
       "      <td>8.912%</td>\n",
       "      <td>10.149%</td>\n",
       "      <td>7.921%</td>\n",
       "      <td>18.91%</td>\n",
       "      <td>15.453%</td>\n",
       "      <td>6.418%</td>\n",
       "      <td>7.968%</td>\n",
       "      <td>7.562%</td>\n",
       "      <td>11.926%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        english   french   german  spanish portuguese esperanto  italian  \\\n",
       "letter                                                                     \n",
       "a        8.167%   7.636%   6.516%  11.525%    14.634%   12.117%  11.745%   \n",
       "b        1.492%   0.901%   1.886%   2.215%     1.043%    0.980%   0.927%   \n",
       "c        2.782%   3.260%   2.732%   4.019%     3.882%    0.776%   4.501%   \n",
       "d        4.253%   3.669%   5.076%   5.010%     4.992%    3.044%   3.736%   \n",
       "e       12.702%  14.715%  16.396%  12.181%    12.570%    8.995%  11.792%   \n",
       "\n",
       "        turkish  swedish  polish   dutch   danish icelandic  finnish   czech  \\\n",
       "letter                                                                         \n",
       "a       11.920%   9.383%  8.965%   7.49%   6.025%   10.110%  12.217%  8.421%   \n",
       "b        2.844%   1.535%  1.482%   1.58%   2.000%    1.043%   0.281%  0.822%   \n",
       "c        0.963%   1.486%  3.988%   1.24%   0.565%       ~0%   0.281%  0.740%   \n",
       "d        4.706%   4.702%  3.293%   5.93%   5.858%    1.575%   1.043%  3.475%   \n",
       "e        8.912%  10.149%  7.921%  18.91%  15.453%    6.418%   7.968%  7.562%   \n",
       "\n",
       "       hungarian  \n",
       "letter            \n",
       "a        10.778%  \n",
       "b         2.647%  \n",
       "c         0.924%  \n",
       "d         2.410%  \n",
       "e        11.926%  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freq_raw.columns = [col.lower().split('[')[0] for col in df_freq_raw.columns]\n",
    "df_freq_raw = df_freq_raw.set_index('letter', drop=True)\n",
    "df_freq_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>letter</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>f</th>\n",
       "      <th>g</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>j</th>\n",
       "      <th>...</th>\n",
       "      <th>û</th>\n",
       "      <th>ŭ</th>\n",
       "      <th>ü</th>\n",
       "      <th>ű</th>\n",
       "      <th>ů</th>\n",
       "      <th>ý</th>\n",
       "      <th>ź</th>\n",
       "      <th>ż</th>\n",
       "      <th>ž</th>\n",
       "      <th>language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.167%</td>\n",
       "      <td>1.492%</td>\n",
       "      <td>2.782%</td>\n",
       "      <td>4.253%</td>\n",
       "      <td>12.702%</td>\n",
       "      <td>2.228%</td>\n",
       "      <td>2.015%</td>\n",
       "      <td>6.094%</td>\n",
       "      <td>6.966%</td>\n",
       "      <td>0.153%</td>\n",
       "      <td>...</td>\n",
       "      <td>~0%</td>\n",
       "      <td>0</td>\n",
       "      <td>~0%</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>english</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.636%</td>\n",
       "      <td>0.901%</td>\n",
       "      <td>3.260%</td>\n",
       "      <td>3.669%</td>\n",
       "      <td>14.715%</td>\n",
       "      <td>1.066%</td>\n",
       "      <td>0.866%</td>\n",
       "      <td>0.737%</td>\n",
       "      <td>7.529%</td>\n",
       "      <td>0.613%</td>\n",
       "      <td>...</td>\n",
       "      <td>0.060%</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>french</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.516%</td>\n",
       "      <td>1.886%</td>\n",
       "      <td>2.732%</td>\n",
       "      <td>5.076%</td>\n",
       "      <td>16.396%</td>\n",
       "      <td>1.656%</td>\n",
       "      <td>3.009%</td>\n",
       "      <td>4.577%</td>\n",
       "      <td>6.550%</td>\n",
       "      <td>0.268%</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.995%</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>german</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.525%</td>\n",
       "      <td>2.215%</td>\n",
       "      <td>4.019%</td>\n",
       "      <td>5.010%</td>\n",
       "      <td>12.181%</td>\n",
       "      <td>0.692%</td>\n",
       "      <td>1.768%</td>\n",
       "      <td>0.703%</td>\n",
       "      <td>6.247%</td>\n",
       "      <td>0.493%</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.012%</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>~0%</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>spanish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14.634%</td>\n",
       "      <td>1.043%</td>\n",
       "      <td>3.882%</td>\n",
       "      <td>4.992%</td>\n",
       "      <td>12.570%</td>\n",
       "      <td>1.023%</td>\n",
       "      <td>1.303%</td>\n",
       "      <td>0.781%</td>\n",
       "      <td>6.186%</td>\n",
       "      <td>0.397%</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.026%</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>portuguese</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 88 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "letter        a       b       c       d        e       f       g       h  \\\n",
       "0        8.167%  1.492%  2.782%  4.253%  12.702%  2.228%  2.015%  6.094%   \n",
       "1        7.636%  0.901%  3.260%  3.669%  14.715%  1.066%  0.866%  0.737%   \n",
       "2        6.516%  1.886%  2.732%  5.076%  16.396%  1.656%  3.009%  4.577%   \n",
       "3       11.525%  2.215%  4.019%  5.010%  12.181%  0.692%  1.768%  0.703%   \n",
       "4       14.634%  1.043%  3.882%  4.992%  12.570%  1.023%  1.303%  0.781%   \n",
       "\n",
       "letter       i       j  ...       û  ŭ       ü  ű  ů    ý  ź  ż  ž    language  \n",
       "0       6.966%  0.153%  ...     ~0%  0     ~0%  0  0    0  0  0  0     english  \n",
       "1       7.529%  0.613%  ...  0.060%  0       0  0  0    0  0  0  0      french  \n",
       "2       6.550%  0.268%  ...       0  0  0.995%  0  0    0  0  0  0      german  \n",
       "3       6.247%  0.493%  ...       0  0  0.012%  0  0  ~0%  0  0  0     spanish  \n",
       "4       6.186%  0.397%  ...       0  0  0.026%  0  0    0  0  0  0  portuguese  \n",
       "\n",
       "[5 rows x 88 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# preparing the data \n",
    "\n",
    "df_freq_raw = df_freq_raw.T\n",
    "df_freq_raw['language'] = df_freq_raw.index\n",
    "df_freq_raw = df_freq_raw.reset_index(drop=True)\n",
    "\n",
    "df_freq_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parsing data to float\n",
    "df_freq = df_freq_raw.drop('language', axis=1).applymap(\n",
    "    lambda x: get_percentage(x)\n",
    ")\n",
    "\n",
    "df_freq['language'] = df_freq_raw['language']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>letter</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>f</th>\n",
       "      <th>g</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>j</th>\n",
       "      <th>...</th>\n",
       "      <th>û</th>\n",
       "      <th>ŭ</th>\n",
       "      <th>ü</th>\n",
       "      <th>ű</th>\n",
       "      <th>ů</th>\n",
       "      <th>ý</th>\n",
       "      <th>ź</th>\n",
       "      <th>ż</th>\n",
       "      <th>ž</th>\n",
       "      <th>language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.08167</td>\n",
       "      <td>0.01492</td>\n",
       "      <td>0.02782</td>\n",
       "      <td>0.04253</td>\n",
       "      <td>0.12702</td>\n",
       "      <td>0.02228</td>\n",
       "      <td>0.02015</td>\n",
       "      <td>0.06094</td>\n",
       "      <td>0.06966</td>\n",
       "      <td>0.00153</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>english</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.07636</td>\n",
       "      <td>0.00901</td>\n",
       "      <td>0.03260</td>\n",
       "      <td>0.03669</td>\n",
       "      <td>0.14715</td>\n",
       "      <td>0.01066</td>\n",
       "      <td>0.00866</td>\n",
       "      <td>0.00737</td>\n",
       "      <td>0.07529</td>\n",
       "      <td>0.00613</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0006</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>french</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.06516</td>\n",
       "      <td>0.01886</td>\n",
       "      <td>0.02732</td>\n",
       "      <td>0.05076</td>\n",
       "      <td>0.16396</td>\n",
       "      <td>0.01656</td>\n",
       "      <td>0.03009</td>\n",
       "      <td>0.04577</td>\n",
       "      <td>0.06550</td>\n",
       "      <td>0.00268</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00995</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>german</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.11525</td>\n",
       "      <td>0.02215</td>\n",
       "      <td>0.04019</td>\n",
       "      <td>0.05010</td>\n",
       "      <td>0.12181</td>\n",
       "      <td>0.00692</td>\n",
       "      <td>0.01768</td>\n",
       "      <td>0.00703</td>\n",
       "      <td>0.06247</td>\n",
       "      <td>0.00493</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00012</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>spanish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.14634</td>\n",
       "      <td>0.01043</td>\n",
       "      <td>0.03882</td>\n",
       "      <td>0.04992</td>\n",
       "      <td>0.12570</td>\n",
       "      <td>0.01023</td>\n",
       "      <td>0.01303</td>\n",
       "      <td>0.00781</td>\n",
       "      <td>0.06186</td>\n",
       "      <td>0.00397</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00026</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>portuguese</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 88 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "letter        a        b        c        d        e        f        g  \\\n",
       "0       0.08167  0.01492  0.02782  0.04253  0.12702  0.02228  0.02015   \n",
       "1       0.07636  0.00901  0.03260  0.03669  0.14715  0.01066  0.00866   \n",
       "2       0.06516  0.01886  0.02732  0.05076  0.16396  0.01656  0.03009   \n",
       "3       0.11525  0.02215  0.04019  0.05010  0.12181  0.00692  0.01768   \n",
       "4       0.14634  0.01043  0.03882  0.04992  0.12570  0.01023  0.01303   \n",
       "\n",
       "letter        h        i        j  ...       û    ŭ        ü    ű    ů    ý  \\\n",
       "0       0.06094  0.06966  0.00153  ...  0.0000  0.0  0.00000  0.0  0.0  0.0   \n",
       "1       0.00737  0.07529  0.00613  ...  0.0006  0.0  0.00000  0.0  0.0  0.0   \n",
       "2       0.04577  0.06550  0.00268  ...  0.0000  0.0  0.00995  0.0  0.0  0.0   \n",
       "3       0.00703  0.06247  0.00493  ...  0.0000  0.0  0.00012  0.0  0.0  0.0   \n",
       "4       0.00781  0.06186  0.00397  ...  0.0000  0.0  0.00026  0.0  0.0  0.0   \n",
       "\n",
       "letter    ź    ż    ž    language  \n",
       "0       0.0  0.0  0.0     english  \n",
       "1       0.0  0.0  0.0      french  \n",
       "2       0.0  0.0  0.0      german  \n",
       "3       0.0  0.0  0.0     spanish  \n",
       "4       0.0  0.0  0.0  portuguese  \n",
       "\n",
       "[5 rows x 88 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freq.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',\n",
       "       'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'à', 'â',\n",
       "       'á', 'å', 'ä', 'ã', 'ą', 'æ', 'œ', 'ç', 'ĉ', 'ć', 'č', 'ď', 'ð', 'è',\n",
       "       'é', 'ê', 'ë', 'ę', 'ě', 'ĝ', 'ğ', 'ĥ', 'î', 'ì', 'í', 'ï', 'ı', 'ĵ',\n",
       "       'ł', 'ľ', 'ñ', 'ń', 'ň', 'ò', 'ö', 'ô', 'ó', 'ő', 'õ', 'ø', 'ř', 'ŝ',\n",
       "       'ş', 'ś', 'š', 'ß', 'ť', 'þ', 'ù', 'ú', 'û', 'ŭ', 'ü', 'ű', 'ů', 'ý',\n",
       "       'ź', 'ż', 'ž', 'language'],\n",
       "      dtype='object', name='letter')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_freq.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',\n",
       "       'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'language'],\n",
       "      dtype='object', name='letter')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# removing not latin characters\n",
    "\n",
    "df_freq = df_freq.drop([\n",
    "    'à', 'â', 'á', 'å', 'ä', 'ã', 'ą', 'æ', 'œ', 'ç', 'ĉ', 'ć', 'č', 'ď',\n",
    "    'ð', 'è', 'é', 'ê', 'ë', 'ę', 'ě', 'ĝ', 'ğ', 'ĥ', 'î', 'ì', 'í', 'ï', \n",
    "    'ı', 'ĵ', 'ł', 'ľ', 'ñ', 'ń', 'ň', 'ò', 'ö', 'ô', 'ó', 'ő', 'õ', 'ø',\n",
    "    'ř', 'ŝ', 'ş', 'ś', 'š', 'ß', 'ť', 'þ', 'ù', 'ú', 'û', 'ŭ', 'ü', 'ű',\n",
    "    'ů', 'ý', 'ź', 'ż', 'ž'\n",
    "], axis=1)\n",
    "\n",
    "df_freq.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(16, 27)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>letter</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>f</th>\n",
       "      <th>g</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>j</th>\n",
       "      <th>...</th>\n",
       "      <th>r</th>\n",
       "      <th>s</th>\n",
       "      <th>t</th>\n",
       "      <th>u</th>\n",
       "      <th>v</th>\n",
       "      <th>w</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "      <th>language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.08167</td>\n",
       "      <td>0.01492</td>\n",
       "      <td>0.02782</td>\n",
       "      <td>0.04253</td>\n",
       "      <td>0.12702</td>\n",
       "      <td>0.02228</td>\n",
       "      <td>0.02015</td>\n",
       "      <td>0.06094</td>\n",
       "      <td>0.06966</td>\n",
       "      <td>0.00153</td>\n",
       "      <td>...</td>\n",
       "      <td>0.05987</td>\n",
       "      <td>0.06327</td>\n",
       "      <td>0.09056</td>\n",
       "      <td>0.02758</td>\n",
       "      <td>0.00978</td>\n",
       "      <td>0.02360</td>\n",
       "      <td>0.00150</td>\n",
       "      <td>0.01974</td>\n",
       "      <td>0.00074</td>\n",
       "      <td>english</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.07636</td>\n",
       "      <td>0.00901</td>\n",
       "      <td>0.03260</td>\n",
       "      <td>0.03669</td>\n",
       "      <td>0.14715</td>\n",
       "      <td>0.01066</td>\n",
       "      <td>0.00866</td>\n",
       "      <td>0.00737</td>\n",
       "      <td>0.07529</td>\n",
       "      <td>0.00613</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06693</td>\n",
       "      <td>0.07948</td>\n",
       "      <td>0.07244</td>\n",
       "      <td>0.06311</td>\n",
       "      <td>0.01838</td>\n",
       "      <td>0.00049</td>\n",
       "      <td>0.00427</td>\n",
       "      <td>0.00128</td>\n",
       "      <td>0.00326</td>\n",
       "      <td>french</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.06516</td>\n",
       "      <td>0.01886</td>\n",
       "      <td>0.02732</td>\n",
       "      <td>0.05076</td>\n",
       "      <td>0.16396</td>\n",
       "      <td>0.01656</td>\n",
       "      <td>0.03009</td>\n",
       "      <td>0.04577</td>\n",
       "      <td>0.06550</td>\n",
       "      <td>0.00268</td>\n",
       "      <td>...</td>\n",
       "      <td>0.07003</td>\n",
       "      <td>0.07270</td>\n",
       "      <td>0.06154</td>\n",
       "      <td>0.04166</td>\n",
       "      <td>0.00846</td>\n",
       "      <td>0.01921</td>\n",
       "      <td>0.00034</td>\n",
       "      <td>0.00039</td>\n",
       "      <td>0.01134</td>\n",
       "      <td>german</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.11525</td>\n",
       "      <td>0.02215</td>\n",
       "      <td>0.04019</td>\n",
       "      <td>0.05010</td>\n",
       "      <td>0.12181</td>\n",
       "      <td>0.00692</td>\n",
       "      <td>0.01768</td>\n",
       "      <td>0.00703</td>\n",
       "      <td>0.06247</td>\n",
       "      <td>0.00493</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06871</td>\n",
       "      <td>0.07977</td>\n",
       "      <td>0.04632</td>\n",
       "      <td>0.02927</td>\n",
       "      <td>0.01138</td>\n",
       "      <td>0.00017</td>\n",
       "      <td>0.00215</td>\n",
       "      <td>0.01008</td>\n",
       "      <td>0.00467</td>\n",
       "      <td>spanish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.14634</td>\n",
       "      <td>0.01043</td>\n",
       "      <td>0.03882</td>\n",
       "      <td>0.04992</td>\n",
       "      <td>0.12570</td>\n",
       "      <td>0.01023</td>\n",
       "      <td>0.01303</td>\n",
       "      <td>0.00781</td>\n",
       "      <td>0.06186</td>\n",
       "      <td>0.00397</td>\n",
       "      <td>...</td>\n",
       "      <td>0.06530</td>\n",
       "      <td>0.06805</td>\n",
       "      <td>0.04336</td>\n",
       "      <td>0.03639</td>\n",
       "      <td>0.01575</td>\n",
       "      <td>0.00037</td>\n",
       "      <td>0.00253</td>\n",
       "      <td>0.00006</td>\n",
       "      <td>0.00470</td>\n",
       "      <td>portuguese</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "letter        a        b        c        d        e        f        g  \\\n",
       "0       0.08167  0.01492  0.02782  0.04253  0.12702  0.02228  0.02015   \n",
       "1       0.07636  0.00901  0.03260  0.03669  0.14715  0.01066  0.00866   \n",
       "2       0.06516  0.01886  0.02732  0.05076  0.16396  0.01656  0.03009   \n",
       "3       0.11525  0.02215  0.04019  0.05010  0.12181  0.00692  0.01768   \n",
       "4       0.14634  0.01043  0.03882  0.04992  0.12570  0.01023  0.01303   \n",
       "\n",
       "letter        h        i        j  ...        r        s        t        u  \\\n",
       "0       0.06094  0.06966  0.00153  ...  0.05987  0.06327  0.09056  0.02758   \n",
       "1       0.00737  0.07529  0.00613  ...  0.06693  0.07948  0.07244  0.06311   \n",
       "2       0.04577  0.06550  0.00268  ...  0.07003  0.07270  0.06154  0.04166   \n",
       "3       0.00703  0.06247  0.00493  ...  0.06871  0.07977  0.04632  0.02927   \n",
       "4       0.00781  0.06186  0.00397  ...  0.06530  0.06805  0.04336  0.03639   \n",
       "\n",
       "letter        v        w        x        y        z    language  \n",
       "0       0.00978  0.02360  0.00150  0.01974  0.00074     english  \n",
       "1       0.01838  0.00049  0.00427  0.00128  0.00326      french  \n",
       "2       0.00846  0.01921  0.00034  0.00039  0.01134      german  \n",
       "3       0.01138  0.00017  0.00215  0.01008  0.00467     spanish  \n",
       "4       0.01575  0.00037  0.00253  0.00006  0.00470  portuguese  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df_freq.shape)\n",
    "df_freq.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data split and model creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((16, 26), (16,))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train data\n",
    "\n",
    "X = df_freq.drop('language', axis=1).values\n",
    "y = df_freq['language']\n",
    "\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        english\n",
       "1         french\n",
       "2         german\n",
       "3        spanish\n",
       "4     portuguese\n",
       "5      esperanto\n",
       "6        italian\n",
       "7        turkish\n",
       "8        swedish\n",
       "9         polish\n",
       "10         dutch\n",
       "11        danish\n",
       "12     icelandic\n",
       "13       finnish\n",
       "14         czech\n",
       "15     hungarian\n",
       "Name: language, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB()"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creating a model\n",
    "\n",
    "model = GaussianNB()\n",
    "model.fit(X, y)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and processing the books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# book process pipeline\n",
    "\n",
    "def process_book(book_path):\n",
    "    raw_book = read_raw_book(book_path)\n",
    "    book = clean_raw_book(raw_book)\n",
    "\n",
    "    chars_count = count_chars(book)\n",
    "    chars_freq_df = get_freq_df(chars_count, book)\n",
    "\n",
    "    return chars_freq_df, book"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'deeplearningbook': (None, None),\n",
       " 'ml_with_sklearn_keras_tf_en': (None, None),\n",
       " 'ml_with_sklearn_keras_tf_pt': (None, None)}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books = [\n",
    "    'deeplearningbook',\n",
    "    'ml_with_sklearn_keras_tf_en',\n",
    "    'ml_with_sklearn_keras_tf_pt'\n",
    "]\n",
    "\n",
    "books_mapper = {book: (None, None) for book in books}\n",
    "books_mapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "book deeplearningbook completed!\n",
      "book ml_with_sklearn_keras_tf_en completed!\n",
      "book ml_with_sklearn_keras_tf_pt completed!\n"
     ]
    }
   ],
   "source": [
    "# reading and processing the books\n",
    "\n",
    "for book_name in books:\n",
    "    path_to_book = f'./books/{book_name}.pdf'\n",
    "    df_book_chars, book = process_book(path_to_book)\n",
    "\n",
    "    books_mapper[book_name] = (df_book_chars, book)\n",
    "    print(f'book {book_name} completed!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "book name = deeplearningbook\n",
      "book header\n",
      "deeplearningiangoodfellowyoshuabengioaaroncourvillecontentswebsiteviiacknowledgmentsviiinotationxiin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>f</th>\n",
       "      <th>g</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>j</th>\n",
       "      <th>...</th>\n",
       "      <th>q</th>\n",
       "      <th>r</th>\n",
       "      <th>s</th>\n",
       "      <th>t</th>\n",
       "      <th>u</th>\n",
       "      <th>v</th>\n",
       "      <th>w</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.082568</td>\n",
       "      <td>0.015325</td>\n",
       "      <td>0.034255</td>\n",
       "      <td>0.032905</td>\n",
       "      <td>0.121933</td>\n",
       "      <td>0.02195</td>\n",
       "      <td>0.021133</td>\n",
       "      <td>0.039973</td>\n",
       "      <td>0.082644</td>\n",
       "      <td>0.001559</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0016</td>\n",
       "      <td>0.064572</td>\n",
       "      <td>0.061125</td>\n",
       "      <td>0.093327</td>\n",
       "      <td>0.027048</td>\n",
       "      <td>0.011791</td>\n",
       "      <td>0.013638</td>\n",
       "      <td>0.006727</td>\n",
       "      <td>0.013264</td>\n",
       "      <td>0.002742</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c         d         e        f         g  \\\n",
       "0  0.082568  0.015325  0.034255  0.032905  0.121933  0.02195  0.021133   \n",
       "\n",
       "          h         i         j  ...       q         r         s         t  \\\n",
       "0  0.039973  0.082644  0.001559  ...  0.0016  0.064572  0.061125  0.093327   \n",
       "\n",
       "          u         v         w         x         y         z  \n",
       "0  0.027048  0.011791  0.013638  0.006727  0.013264  0.002742  \n",
       "\n",
       "[1 rows x 26 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "book name = ml_with_sklearn_keras_tf_en\n",
      "book header\n",
      "aureliengeronhandsonmachinelearningwithscikitlearnkerasandtensorflowconceptstoolsandtechniquestobuil\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>f</th>\n",
       "      <th>g</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>j</th>\n",
       "      <th>...</th>\n",
       "      <th>q</th>\n",
       "      <th>r</th>\n",
       "      <th>s</th>\n",
       "      <th>t</th>\n",
       "      <th>u</th>\n",
       "      <th>v</th>\n",
       "      <th>w</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.081389</td>\n",
       "      <td>0.012148</td>\n",
       "      <td>0.034525</td>\n",
       "      <td>0.031778</td>\n",
       "      <td>0.119757</td>\n",
       "      <td>0.021384</td>\n",
       "      <td>0.022313</td>\n",
       "      <td>0.039544</td>\n",
       "      <td>0.081464</td>\n",
       "      <td>0.00136</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001707</td>\n",
       "      <td>0.065115</td>\n",
       "      <td>0.074335</td>\n",
       "      <td>0.096437</td>\n",
       "      <td>0.02907</td>\n",
       "      <td>0.009148</td>\n",
       "      <td>0.012042</td>\n",
       "      <td>0.004582</td>\n",
       "      <td>0.016585</td>\n",
       "      <td>0.001507</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c         d         e         f         g  \\\n",
       "0  0.081389  0.012148  0.034525  0.031778  0.119757  0.021384  0.022313   \n",
       "\n",
       "          h         i        j  ...         q         r         s         t  \\\n",
       "0  0.039544  0.081464  0.00136  ...  0.001707  0.065115  0.074335  0.096437   \n",
       "\n",
       "         u         v         w         x         y         z  \n",
       "0  0.02907  0.009148  0.012042  0.004582  0.016585  0.001507  \n",
       "\n",
       "[1 rows x 26 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "book name = ml_with_sklearn_keras_tf_pt\n",
      "book header\n",
      "maosaobraaprendizadodemaquinacomscikitlearntensorflowconceitosferramentasetecnicasparaaconstrucaodes\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>e</th>\n",
       "      <th>f</th>\n",
       "      <th>g</th>\n",
       "      <th>h</th>\n",
       "      <th>i</th>\n",
       "      <th>j</th>\n",
       "      <th>...</th>\n",
       "      <th>q</th>\n",
       "      <th>r</th>\n",
       "      <th>s</th>\n",
       "      <th>t</th>\n",
       "      <th>u</th>\n",
       "      <th>v</th>\n",
       "      <th>w</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.126552</td>\n",
       "      <td>0.008377</td>\n",
       "      <td>0.043037</td>\n",
       "      <td>0.05266</td>\n",
       "      <td>0.1229</td>\n",
       "      <td>0.012133</td>\n",
       "      <td>0.014027</td>\n",
       "      <td>0.008354</td>\n",
       "      <td>0.068913</td>\n",
       "      <td>0.002889</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007424</td>\n",
       "      <td>0.069639</td>\n",
       "      <td>0.072776</td>\n",
       "      <td>0.053709</td>\n",
       "      <td>0.037647</td>\n",
       "      <td>0.014123</td>\n",
       "      <td>0.001585</td>\n",
       "      <td>0.005018</td>\n",
       "      <td>0.002081</td>\n",
       "      <td>0.006016</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          a         b         c        d       e         f         g  \\\n",
       "0  0.126552  0.008377  0.043037  0.05266  0.1229  0.012133  0.014027   \n",
       "\n",
       "          h         i         j  ...         q         r         s         t  \\\n",
       "0  0.008354  0.068913  0.002889  ...  0.007424  0.069639  0.072776  0.053709   \n",
       "\n",
       "          u         v         w         x         y         z  \n",
       "0  0.037647  0.014123  0.001585  0.005018  0.002081  0.006016  \n",
       "\n",
       "[1 rows x 26 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# displaying the frequencies of the books\n",
    "\n",
    "for book_name, (df_book_freq, book) in books_mapper.items():\n",
    "    print('book name =', book_name)\n",
    "    \n",
    "    print('book header')\n",
    "    print(book[:100], end='\\n')\n",
    "    \n",
    "    display(df_book_freq)\n",
    "    print()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making predictions using the `GaussianNB` model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.08256754 0.01532465 0.03425492 0.03290464 0.12193262 0.02194958\n",
      "  0.02113298 0.03997333 0.08264357 0.00155937 0.0053039  0.04147056\n",
      "  0.02801306 0.07572401 0.07221854 0.02693985 0.00160031 0.0645723\n",
      "  0.06112532 0.09332667 0.02704805 0.01179139 0.01363807 0.00672656\n",
      "  0.01326376 0.0027415 ]]\n",
      "\n",
      " prediction = ['english']\n"
     ]
    }
   ],
   "source": [
    "sample_dl = books_mapper['deeplearningbook'][0].values\n",
    "\n",
    "print(sample_dl)\n",
    "print('\\n prediction =', model.predict(sample_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.08138934 0.01214779 0.03452485 0.03177776 0.11975677 0.02138444\n",
      "  0.02231311 0.03954356 0.08146441 0.00135964 0.00555812 0.04709804\n",
      "  0.02776557 0.06914701 0.07087367 0.02335578 0.0017072  0.06511536\n",
      "  0.07433533 0.09643713 0.02906961 0.00914768 0.01204213 0.00458218\n",
      "  0.01658538 0.001507  ]]\n",
      "\n",
      " prediction = ['english']\n"
     ]
    }
   ],
   "source": [
    "sample_ml_en = books_mapper['ml_with_sklearn_keras_tf_en'][0].values\n",
    "\n",
    "print(sample_ml_en)\n",
    "print('\\n prediction =', model.predict(sample_ml_en))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.12655198 0.00837724 0.04303695 0.05265958 0.12290006 0.01213303\n",
      "  0.01402709 0.00835416 0.06891314 0.00288898 0.00143122 0.03370864\n",
      "  0.04681121 0.05848833 0.09898603 0.03071232 0.00742387 0.06963914\n",
      "  0.07277627 0.05370876 0.0376468  0.01412289 0.00158473 0.0050185\n",
      "  0.00208104 0.00601573]]\n",
      "\n",
      " prediction = ['portuguese']\n"
     ]
    }
   ],
   "source": [
    "sample_ml_pt = books_mapper['ml_with_sklearn_keras_tf_pt'][0].values\n",
    "\n",
    "print(sample_ml_pt)\n",
    "print('\\n prediction =', model.predict(sample_ml_pt))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making predictions using `mean_squared_error`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['english', 'french', 'german', 'spanish', 'portuguese',\n",
       "       'esperanto', 'italian', 'turkish', 'swedish', 'polish', 'dutch',\n",
       "       'danish', 'icelandic', 'finnish', 'czech', 'hungarian'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "languages = df_freq['language'].values\n",
    "languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'english'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# deep learning book [english]\n",
    "\n",
    "mse_predict(\n",
    "    sample=sample_dl,\n",
    "    df_freq=df_freq,\n",
    "    X=X\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'english'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ml with sklearn, keras and tf [english]\n",
    "\n",
    "mse_predict(\n",
    "    sample=sample_ml_en,\n",
    "    df_freq=df_freq,\n",
    "    X=X\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'portuguese'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ml with sklearn, keras and tf [portuguese]\n",
    "\n",
    "mse_predict(\n",
    "    sample=sample_ml_pt,\n",
    "    df_freq=df_freq,\n",
    "    X=X\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}