Neural_Network_Multiple_Hidden_Layer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network with Multiple Hidden Layer "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wine Quality Dataset\n",
    "\n",
    "Wine Quality Dataset contains of 720 samples, 11 features and 1 column for 4 quality class labels. Class labels are defined as 0, 1, 2 and 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.478873</td>\n",
       "      <td>0.236364</td>\n",
       "      <td>0.253012</td>\n",
       "      <td>0.025836</td>\n",
       "      <td>0.141618</td>\n",
       "      <td>0.141869</td>\n",
       "      <td>0.277273</td>\n",
       "      <td>0.955745</td>\n",
       "      <td>0.908377</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.739437</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.366197</td>\n",
       "      <td>0.140909</td>\n",
       "      <td>0.198795</td>\n",
       "      <td>0.024316</td>\n",
       "      <td>0.080925</td>\n",
       "      <td>0.044983</td>\n",
       "      <td>0.134091</td>\n",
       "      <td>0.952617</td>\n",
       "      <td>0.863874</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.838028</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.556338</td>\n",
       "      <td>0.463636</td>\n",
       "      <td>0.204819</td>\n",
       "      <td>0.039514</td>\n",
       "      <td>0.141618</td>\n",
       "      <td>0.044983</td>\n",
       "      <td>0.306818</td>\n",
       "      <td>0.956082</td>\n",
       "      <td>0.808901</td>\n",
       "      <td>0.472222</td>\n",
       "      <td>0.704225</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.450704</td>\n",
       "      <td>0.254545</td>\n",
       "      <td>0.246988</td>\n",
       "      <td>0.103343</td>\n",
       "      <td>0.130058</td>\n",
       "      <td>0.211073</td>\n",
       "      <td>0.490909</td>\n",
       "      <td>0.957863</td>\n",
       "      <td>0.808901</td>\n",
       "      <td>0.425926</td>\n",
       "      <td>0.661972</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.485915</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.168675</td>\n",
       "      <td>0.066869</td>\n",
       "      <td>0.083815</td>\n",
       "      <td>0.048443</td>\n",
       "      <td>0.243182</td>\n",
       "      <td>0.956197</td>\n",
       "      <td>0.848168</td>\n",
       "      <td>0.425926</td>\n",
       "      <td>0.732394</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0       0.478873          0.236364     0.253012        0.025836   0.141618   \n",
       "1       0.366197          0.140909     0.198795        0.024316   0.080925   \n",
       "2       0.556338          0.463636     0.204819        0.039514   0.141618   \n",
       "3       0.450704          0.254545     0.246988        0.103343   0.130058   \n",
       "4       0.485915          0.136364     0.168675        0.066869   0.083815   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide   density        pH  sulphates  \\\n",
       "0             0.141869              0.277273  0.955745  0.908377   0.444444   \n",
       "1             0.044983              0.134091  0.952617  0.863874   0.777778   \n",
       "2             0.044983              0.306818  0.956082  0.808901   0.472222   \n",
       "3             0.211073              0.490909  0.957863  0.808901   0.425926   \n",
       "4             0.048443              0.243182  0.956197  0.848168   0.425926   \n",
       "\n",
       "    alcohol  quality_class  \n",
       "0  0.739437              3  \n",
       "1  0.838028              3  \n",
       "2  0.704225              1  \n",
       "3  0.661972              1  \n",
       "4  0.732394              3  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "data = pd.read_csv(\"wine_quality_white_modified.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.478873</td>\n",
       "      <td>0.236364</td>\n",
       "      <td>0.253012</td>\n",
       "      <td>0.025836</td>\n",
       "      <td>0.141618</td>\n",
       "      <td>0.141869</td>\n",
       "      <td>0.277273</td>\n",
       "      <td>0.955745</td>\n",
       "      <td>0.908377</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.739437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.366197</td>\n",
       "      <td>0.140909</td>\n",
       "      <td>0.198795</td>\n",
       "      <td>0.024316</td>\n",
       "      <td>0.080925</td>\n",
       "      <td>0.044983</td>\n",
       "      <td>0.134091</td>\n",
       "      <td>0.952617</td>\n",
       "      <td>0.863874</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.838028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.556338</td>\n",
       "      <td>0.463636</td>\n",
       "      <td>0.204819</td>\n",
       "      <td>0.039514</td>\n",
       "      <td>0.141618</td>\n",
       "      <td>0.044983</td>\n",
       "      <td>0.306818</td>\n",
       "      <td>0.956082</td>\n",
       "      <td>0.808901</td>\n",
       "      <td>0.472222</td>\n",
       "      <td>0.704225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.450704</td>\n",
       "      <td>0.254545</td>\n",
       "      <td>0.246988</td>\n",
       "      <td>0.103343</td>\n",
       "      <td>0.130058</td>\n",
       "      <td>0.211073</td>\n",
       "      <td>0.490909</td>\n",
       "      <td>0.957863</td>\n",
       "      <td>0.808901</td>\n",
       "      <td>0.425926</td>\n",
       "      <td>0.661972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.485915</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.168675</td>\n",
       "      <td>0.066869</td>\n",
       "      <td>0.083815</td>\n",
       "      <td>0.048443</td>\n",
       "      <td>0.243182</td>\n",
       "      <td>0.956197</td>\n",
       "      <td>0.848168</td>\n",
       "      <td>0.425926</td>\n",
       "      <td>0.732394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0       0.478873          0.236364     0.253012        0.025836   0.141618   \n",
       "1       0.366197          0.140909     0.198795        0.024316   0.080925   \n",
       "2       0.556338          0.463636     0.204819        0.039514   0.141618   \n",
       "3       0.450704          0.254545     0.246988        0.103343   0.130058   \n",
       "4       0.485915          0.136364     0.168675        0.066869   0.083815   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide   density        pH  sulphates  \\\n",
       "0             0.141869              0.277273  0.955745  0.908377   0.444444   \n",
       "1             0.044983              0.134091  0.952617  0.863874   0.777778   \n",
       "2             0.044983              0.306818  0.956082  0.808901   0.472222   \n",
       "3             0.211073              0.490909  0.957863  0.808901   0.425926   \n",
       "4             0.048443              0.243182  0.956197  0.848168   0.425926   \n",
       "\n",
       "    alcohol  \n",
       "0  0.739437  \n",
       "1  0.838028  \n",
       "2  0.704225  \n",
       "3  0.661972  \n",
       "4  0.732394  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = data.copy()\n",
    "df2.drop('quality_class', axis=1, inplace=True)\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fixed acidity           0.070423\n",
      "volatile acidity        0.110227\n",
      "citric acid             0.066265\n",
      "residual sugar          0.103343\n",
      "chlorides               0.043353\n",
      "free sulfur dioxide     0.076125\n",
      "total sulfur dioxide    0.131818\n",
      "density                 0.003970\n",
      "pH                      0.057592\n",
      "sulphates               0.159722\n",
      "alcohol                 0.154930\n",
      "quality_class           1.500000\n",
      "dtype: float64\n",
      "(570, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.478873</td>\n",
       "      <td>0.236364</td>\n",
       "      <td>0.253012</td>\n",
       "      <td>0.025836</td>\n",
       "      <td>0.141618</td>\n",
       "      <td>0.141869</td>\n",
       "      <td>0.277273</td>\n",
       "      <td>0.955745</td>\n",
       "      <td>0.908377</td>\n",
       "      <td>0.444444</td>\n",
       "      <td>0.739437</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.556338</td>\n",
       "      <td>0.463636</td>\n",
       "      <td>0.204819</td>\n",
       "      <td>0.039514</td>\n",
       "      <td>0.141618</td>\n",
       "      <td>0.044983</td>\n",
       "      <td>0.306818</td>\n",
       "      <td>0.956082</td>\n",
       "      <td>0.808901</td>\n",
       "      <td>0.472222</td>\n",
       "      <td>0.704225</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.450704</td>\n",
       "      <td>0.254545</td>\n",
       "      <td>0.246988</td>\n",
       "      <td>0.103343</td>\n",
       "      <td>0.130058</td>\n",
       "      <td>0.211073</td>\n",
       "      <td>0.490909</td>\n",
       "      <td>0.957863</td>\n",
       "      <td>0.808901</td>\n",
       "      <td>0.425926</td>\n",
       "      <td>0.661972</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.485915</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.168675</td>\n",
       "      <td>0.066869</td>\n",
       "      <td>0.083815</td>\n",
       "      <td>0.048443</td>\n",
       "      <td>0.243182</td>\n",
       "      <td>0.956197</td>\n",
       "      <td>0.848168</td>\n",
       "      <td>0.425926</td>\n",
       "      <td>0.732394</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.507042</td>\n",
       "      <td>0.218182</td>\n",
       "      <td>0.204819</td>\n",
       "      <td>0.016717</td>\n",
       "      <td>0.130058</td>\n",
       "      <td>0.010381</td>\n",
       "      <td>0.145455</td>\n",
       "      <td>0.954109</td>\n",
       "      <td>0.845550</td>\n",
       "      <td>0.472222</td>\n",
       "      <td>0.802817</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0       0.478873          0.236364     0.253012        0.025836   0.141618   \n",
       "1       0.556338          0.463636     0.204819        0.039514   0.141618   \n",
       "2       0.450704          0.254545     0.246988        0.103343   0.130058   \n",
       "3       0.485915          0.136364     0.168675        0.066869   0.083815   \n",
       "4       0.507042          0.218182     0.204819        0.016717   0.130058   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide   density        pH  sulphates  \\\n",
       "0             0.141869              0.277273  0.955745  0.908377   0.444444   \n",
       "1             0.044983              0.306818  0.956082  0.808901   0.472222   \n",
       "2             0.211073              0.490909  0.957863  0.808901   0.425926   \n",
       "3             0.048443              0.243182  0.956197  0.848168   0.425926   \n",
       "4             0.010381              0.145455  0.954109  0.845550   0.472222   \n",
       "\n",
       "    alcohol  quality_class  \n",
       "0  0.739437              3  \n",
       "1  0.704225              1  \n",
       "2  0.661972              1  \n",
       "3  0.732394              3  \n",
       "4  0.802817              1  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABI8AAAJNCAYAAAC4BVWHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdf5Cc910n+PczbgWvk5BYbTu2CJrgUXbrcGCPJAeJs+GYcaRKAgtVrq27TZNaWFOAjlPuUrvSCJvdrV2WnM6yqNslhFO4Qwd3S2c3f7iOkCirGZhxfthesubqDjwLgRmFGRw52PQkThytidv93B/6gdrRjGRpWt/umderSoU/o+HxGzejab3n83yfqq7rAAAAAMDFjJUOAAAAAMDwUh4BAAAAsCblEQAAAABrUh4BAAAAsCblEQAAAABrUh4BAAAAsKZG6QAv1U033VS/7nWvKx0DAAAAYNP4/d///b+s6/rmi/3eyJVHr3vd6/LYY4+VjgEAAACwaVRVtbzW77ltDQAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1Daw8qqrqWFVVT1VV9fgav19VVfVLVVUtVlX1B1VVvXFQWeBydDqd7N+/P6urq6WjALxk/gwbPl6T4eR1gUvzdQK82CA3j349yTvX+f13JXn92V8/leR/HWAWuKR2u52FhYW02+3SUQBeMn+GDR+vyXDyusCl+ToBXmxg5VFd159Osl5V/SNJ/s/6jP+Q5NVVVd02qDywnk6nk9nZ2dR1nZmZGT9lAUaKP8OGj9dkOHld4NI6nU5mZmZS13VOnDjh6wRIkjQK/ru/LcmfXzA/cfZjT5aJw1bWbrfT6/WSJL1eL+12O/v27SucCuDy+DNs+HhNhpPXBS6t3W6n2+0mSbrdrq8TNpWjR49maWlpw6536tSpJMmOHTs27JpJMjExkb17927oNa9WyQOzq4t8rL7oJ1bVT1VV9VhVVY89/fTTA47FVjQ/P9/3TXJubq5wIoDL58+w4eM1GU5eF7i0ubm51PWZv5bVdZ3f/d3fLZwIhtdzzz2X5557rnSMa6Lk5tETSb79gvm1SU5d7BPruv7VJL+aJG9+85svWjDB1ZicnMyJEyfS7XbTaDQyNTVVOhLAZfNn2PDxmgwnrwtc2s0335yVlZXz8y233FIwDWysjd7mOXDgQJLkgQce2NDrDqOSm0cfS/IPzj517S1Jnqnr2i1rFNFqtTI2dubLYWxsLK1Wq3AigMvnz7Dh4zUZTl4XuLQX3+nx1FNPFUoCDJOBlUdVVX0kyaNJ/lZVVU9UVfUTVVXtrarqXNV3PMnJJItJ/rckPzOoLHApzWYzu3fvTlVV2bNnT7Zv3146EsBl82fY8PGaDCevC1za1NRUqurMCSNVVeWuu+4qnAgYBgO7ba2u6/dc4vfrJP/9oP798FK1Wq0sLy/7KSQwkvwZNny8JsPJ6wLra7VamZmZyfPPP59t27b5WgGSJNW5w9BGxZvf/Ob6scceKx0DAABgU/rgBz+Y48eP5wd/8Ac9aQ3WsdnOPKqq6vfrun7zxX6v5IHZAAAADBkbesCLKY8AAAA4r9ls5siRI6VjAEOk5NPWAAAAABhyyiMAAADO63Q62b9/f1ZXV0tHAYaE8ggAAIDz2u12FhYW0m63S0cBhoTyCAAAgCRnto5mZmZS13VOnDhh+whIojwCAADgrHa7nW63myTpdru2j4AkyiMAAADOmpubS13XSZK6rvO7v/u7hRMBw0B5BAAAQJLk5ptv7ptvueWWQkmAYaI8AgAAIEny9NNP981PPfVUoSTAMFEeAQAAkCSZmppKVVVJkqqqctdddxVOBAyDRukAAADA6Dh69GiWlpY27HqnTp1KkuzYsWPDrjkxMZG9e/du2PW2klarlZmZmTz//PNpNBpptVqlIwFDwOYRAABQzHPPPZfnnnuudAzOajab5885uuWWW7J9+/bCiYBhYPMIAAC4bBu90XPgwIEkyQMPPLCh1+XKdDqd89tgp06dyurqqgIJUB4BAMBmtdG3mA3CyZMnk/x1iTSstsqtcMeOHUtd10mSuq5z7Nix7N+/v3AqoDTlEQAAbFKf+cxn0lldTbZ9S+koa+s+nyT5w8//aeEg63j+r3Lq1KktUR499NBDffP8/LzyCFAeAQAAjLKN3DB74YUXvmneqK2wrbK9BZuR8ggAADapt7/97SNz29rtt99eOMn6JiYmSke4Jl796lfnK1/5St8MoDwCAIBNahBbHqNwjtJW23DZyP9bO51OfvRHfzRJMjY2ll/5lV9xYDaQsdIBAACArev666/P9ddfXzoGZzWbzfPbRlNTU4ojIInNIwAA4CXYShs9W9Wtt96ab3zjG7nnnntKRwGGhM0jAAAAztu2bVsmJiZsHQHnKY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCAAAAIA1KY8AAAAAWJPyCM7qdDrZv39/VldXS0cBAACAoaE8grPa7XYWFhbSbrdLRwEAAIChoTyCnNk6mp2dTV3XmZmZsX0EAAAAZymPIGe2jnq9XpKk1+vZPgIAAICzlEeQZH5+Pt1uN0nS7XYzNzdXOBEAAAAMB+URJJmcnEyj0UiSNBqNTE1NFU4EAAAAw0F5BElarVbGxs58OYyNjaXVahVOBAAAAMNBeQRJms1mdu/enaqqsmfPnmzfvr10JAAAABgKjdIBYFi0Wq0sLy/bOgIAAIALKI/grGazmSNHjpSOAQAAAEPFbWsAAAAArEl5BAAAAMCalEcAAAAArEl5BAAAAMCalEcAAAAArEl5BAAAAMCalEcAAAAw5BYXF3P33Xfn5MmTpaOwBSmPAAAAYMgdPnw4p0+fzv333186CluQ8ggAAACG2OLiYlZWVpIky8vLto+45pRHAAAAMMQOHz7cN9s+4lpTHgEAAMAQO7d1dM7y8nKhJGxVyiMAAAAYYjt37uybx8fHCyVhq1IeAQAAwBCbnp7umw8ePFgoCVuV8ggAAACANSmPAAAAYIg5MJvSlEcAAAAwxByYTWmN0gEAAAC2kqNHj2Zpaal0jDWdPHkySXLgwIHCSdY3MTGRvXv3lo5xTezcubOvQHJgNtea8ggAAOAaWlpayuc/v5TtzZ2X/uQC6vplSZKn//L5wknWttpZufQnbSLT09PZt2/f+dmB2VxryiMAAIBrbHtzZ979Q/+kdIyRdfzjv1A6wjW1a9eu89tH4+Pjuf3220tHYotx5hEAAAAMuenp6dxwww22jijC5hEAAAAMuV27duXBBx8sHYMtyuYRAAAAAGtSHgEAAACwJuURnNXpdLJ///6srq6WjgIAAABDQ3kEZ7Xb7SwsLKTdbpeOAgAAAENDeQQ5s3U0Ozubuq4zMzNj+wgAAADOUh5Bzmwd9Xq9JEmv17N9BAAAAGcpjyDJ/Px8ut1ukqTb7WZubq5wIoCXxrltAAAMivIIkkxOTqbRaCRJGo1GpqamCicCeGmc2wYAwKAojyBJq9XK2NiZL4exsbG0Wq3CiQAun3Pb4PLZ0gNG1eLiYu6+++6cPHmydBS2oEbpAFtVp9PJoUOHct9992X79u2l42x5zWYzu3fvzvHjx7Nnzx6vCTBSLnZu2759+wqnguF04ZaerxNglBw+fDinT5/O/fffnw9/+MOl41wTR48ezdLSUukYazpX5B04cKBwkvVNTExk7969V3UN5VEh3rgMn1arleXlZVtHwMi52LltvrfAN3vxll6r1fIDI4o4depUvvq10zn+8V8oHWVkdTrLef4bN5SOcc0sLi5mZWUlSbK8vJyTJ0/m9ttvL5xq8JaWlrL0R3+ana+8tXSUi3pZ98zdK88/8bXCSda28rUvbch1lEcFeOMCwEaanJzMiRMn0u12ndsG67ClB4yqw4cP981bafto5ytvzb3fd0/pGCPr0O8d25DrKI8K8MZlONkGA0ZVq9XK7OxsEue2wXps6TEsduzYkW1/+Xze/UP/pHSUkXX847+Qm2/aVjrGNXNu6+ic5eXlQknYqhyYXYDHwg8fh80Co+zcuW1VVTm3Ddbh6arAqNq5c2ffPD4+XigJW5XyqABvXIbPxbbBAEZJq9XKHXfcYesI1uHpqsComp6e7psPHjxYKAlblfKoAG9cho9tMGDUNZvNHDlyxNYRrMOWHjCqdu3adX77aHx8fEscls1wUR4V4I3L8LENBgBbgy09YFRNT0/nhhtusHVEEcqjQrxxGS62wQBga7ClB4yqXbt25cEHH7R1RBHKo0K8cRkutsEAAADg4hqlA8CwaLVaWV5etnUEAAAAF1AewVnntsEAAACAv+a2NQAAAADWpDwCAACAIdfpdLJ///6srq6WjsIW5LY1AACAa2y1s5LjH/+F0jEu6qvP/EWS5Ftf9ZrCSda22lnJzTdNlI5xTbXb7SwsLKTdbmffvn2l47DFKI8AAACuoYmJ4S49vvbVbyRJbr5pW+Eka7v5pomh/++4kTqdTmZnZ1PXdWZmZtJqtTwhmmtqoOVRVVXvTPKvk1yX5H+v6/p/ftHv70zyG0leffZzfrau6+ODzAQAAFDS3r17S0dY14EDB5IkDzzwQOEknNNut9Pr9ZIkvV7P9hHX3MDOPKqq6rokH0ryriTfmeQ9VVV954s+7Z8k+Whd19+T5O8n+ZVB5QEAAIBRND8/n263myTpdruZm5srnIitZpAHZn9vksW6rk/Wdf2NJP82yY+86HPqJN969p9fleTUAPMAAADAyJmcnEyjcebGoUajkampqcKJ2GoGWR59W5I/v2B+4uzHLvTPk7y3qqonkhxP8r4B5gEAAICR02q1+m5ba7VahROx1QyyPKou8rH6RfN7kvx6XdevTfLuJP9XVVXflKmqqp+qquqxqqoee/rppwcQFQAAAICLGWR59ESSb79gfm2++ba0n0jy0SSp6/rRJNcnuenFF6rr+lfrun5zXddvvvnmmwcUFwAAAIZPu91OVZ3Zz6iqKu12u3AitppBlkf/Mcnrq6r6jqqqXpYzB2J/7EWfs5LkriSpquq/yJnyyGoRAAAAnDU/P58XXnghSfLCCy84MJtrbmDlUV3X3ST7kpxI8kc581S1haqqfr6qqh8++2n/OMlPVlX1/yX5SJIfr+v6xbe2AQAAwJblwGxKawzy4nVdH8+Zg7Av/Ng/u+Cf/1OStw0yAwAAAIyyVquV2dnZJMnY2JgDs7nmBnnbGgAAAHCVms1mdu/enaqqsmfPnmzfvr10JLaYgW4eAQAAAFev1WpleXnZ1hFF2DwCAACAIfflL385J0+ezFe+8pXSUdiClEcAAAAw5A4fPpzTp0/n/vvvLx2FLUh5BAAAAENscXExKysrSZLl5eWcPHmycCK2GuURAAAADLHDhw/3zbaPuNaURwAAADDEzm0dnbO8vFwoCVuV8ggAAACG2M6dO/vm8fHxQknYqpRHAAAAMMSmp6f75oMHDxZKwlalPAIAAIAhtmvXrvPbR+Pj47n99tsLJ2KrUR4BAADAkJuens4NN9xg64giGqUDAAAAAOvbtWtXHnzwwdIx2KJsHgEAAACwJptHAAAAwNA5depUTn/t2Rz6vWOlo4ys5a89mRtOfe2qr6M8AgAAgA129OjRLC0tbdj1Tp06lSTZsWPHhl1zYmIie/fu3bDrsXkpjwAAAGDIPffcc6UjXHM7duzI872v5d7vu6d0lJF16PeOZduOV171dZRHAAAAsME2eqPnwIEDSZIHHnhgQ68Ll8OB2QAAAACsSXkEAAAAwJqURwAAAACsSXkEAAAAwJqURwAAAACsSXkEAAAAwJoapQMAAABw5Y4ePZqlpaUNu97JkyeT/PWj4TfKxMTEhj++Hrg2lEcAAACcd/3115eOAAwZ5REAAMAIs80DDJozjwAAAABYk/IIAAAAgDUpjwrpdDrZv39/VldXS0cBAAAAWJPyqJB2u52FhYW02+3SUQAAAADWpDwqoNPpZHZ2NnVdZ2ZmxvYRAAAAMLSURwW02+30er0kSa/Xs30EAAAADC3lUQHz8/PpdrtJkm63m7m5ucKJAAAAAC5OeVTAW9/61r75zjvvLJQEAAAAYH3KIwAAAADWpDwq4NFHH+2bH3nkkUJJAAAAANanPCpgcnIy1113XZLkuuuuy9TUVOFEAAAAABenPCqg1Wr1lUetVqtwIgAAAICLUx4V0Gw28/a3vz1J8v3f//3Zvn174UQAjLpOp5P9+/dndXW1dBQAADYZ5REAbALtdjsLCwtpt9ulowAAsMkojwrodDr5zGc+kyT59Kc/7afEAFyVTqeT2dnZ1HWdmZkZ31cAANhQyqMC2u12er1ekqTX6/kpMQBXxfcVuHxu8QSAl055VMD8/Hy63W6SpNvtZm5urnAiAEaZ7ytw+dziCQAvnfKogMnJyVRVlSSpqipTU1OFEwEwyiYnJ9NoNJIkjUbD9xVYg1s8AeDKKI8KeNe73pW6rpMkdV3n3e9+d+FEAIyyVquVsbEz39LHxsbSarUKJ4Lh5BZPALgyyqMCPvnJT/bNx48fL5SECy0uLubuu+/OyZMnS0cBeEmazWZ2796dqqqyZ8+ebN++vXQkGEpu8QSAK6M8KmB+fr5v9sZlOBw+fDinT5/O/fffXzoKwEvWarVyxx132DqCdbjFEwCujPKogDe84Q1983d913cVSsI5i4uLWVlZSZIsLy/bPgJGTrPZzJEjR2wdwTrc4gkAV0Z5VMDCwkLf/PjjjxdKwjmHDx/um20fAcDm4xZPALgyjdIBtqKvf/3r685ce+e2js5ZXl4ulAQAGKRWq5Xl5WVbRwDwEtg8KuAVr3jFujPX3s6dO/vm8fHxQkkAgEFyiycAvHTKowLuvffevvnnfu7nCiXhnJ/+6Z/um/fu3VsoCQAAAAwXt60V8KY3vSmveMUr8uyzz+YVr3hFvud7vqd0pC3vkUce6ZsffvhhrwsAAEBhK1/7Ug793rHSMS7qL053kiSvuaFZOMnaVr72pUzklVd9HeVRIffee2/+6T/9p7aOhsT8/HzfPDc3l3379hVKAwAAwMTEROkI6/rGyaeTJNtee/XlzKBM5JUb8t9ReVTIm970phw/frx0DM6anJzMiRMn0u1202g0MjU1VToSAADAljbsx4kcOHAgSfLAAw8UTjJ4zjyCnHnyytjYmS+HsbExT2ABAACAs5RHkDNPXtm9e3eqqsqePXs8gQUAAADOctsanNVqtbK8vGzrCAAAAC5g8wgAAACANSmPCpmfn8873/nOfPrTny4dhbPa7XYWFhbSbrdLRwEAAIChoTwq5Bd/8ReTJIcPHy6chCTpdDqZnZ1NXdeZmZnJ6upq6UgAAAAwFJRHBczPz6fb7SZJut2u7aMh0G630+v1kiS9Xs/2EQAAAJylPCrg3NbRObaPyntxoTc3N1c4EQAAAAwH5VEB50qKtWauvcnJyTQaZx4+2Gg0MjU1VTgRAAAADAflUQHXXXfdujPXXqvVytjYmS+HsbGxtFqtwokAAABgOCiPCviu7/quvvlv/+2/XSgJ5zSbzezevTtVVWXPnj3Zvn176UgAAAAwFBqlA2xFf/Inf9I3//Ef/3GhJFyo1WpleXnZ1hEAAABcwOZRAZOTk32z83WGQ7PZzJEjR2wdAQAAwAWURwW8/OUv75tf+cpXFkoCAAAAsD7lUQEf/ehH++aPfOQjhZJwoU6nk/3792d1dbV0FAAAABgayiM4q91uZ2FhIe12u3QUAAAAGBrKI8iZraPZ2dnUdZ2ZmRnbRwAAAHCW8qiAV73qVX3zq1/96kJJOKfdbqfX6yVJer2e7SMAAAA4S3lUwDPPPNM3f+UrXymUhHPm5+fT7XaTJN1uN3Nzc4UTAQAAwHBQHhWwc+fOvnl8fLxQEs6ZnJxMo9FIkjQajUxNTRVOBAAAAMNBeVTAe97znr75R3/0Rwsl4ZxWq5WxsTNfDmNjY2m1WoUTAQAAwHBQHhXwkY98pG/+zd/8zUJJOKfZbGb37t2pqip79uzJ9u3bS0cCAACAodAoHWArWllZ6ZuXl5cLJeFCrVYry8vLto4AAADgAjaPCrj11lv75ttuu61QEi7UbDZz5MgRW0cAAABwAeURAAAAAGtSHhXwpS99qW9+8sknCyUBAAAAWJ/yqICdO3f2zePj44WScKFOp5P9+/dndXW1dBQAYEB8vweAl055VMBP//RP98179+4tlIQLtdvtLCwspN1ul44CAAyI7/cA8NIpjwp45JFH+uaHH364UBLO6XQ6mZ2dTV3XmZmZ8dNIANiEfL8HgCujPCpgfn6+b56bmyuUhHPa7XZ6vV6SpNfr+WkkAGxCvt8DwJVRHhUwOTmZRqORJGk0GpmamiqciPn5+XS73SRJt9tV6AHAJuT7PQBcmUbpAKPi6NGjWVpa2pBrPf/88+ffuLzwwgtZXFzMgQMHrvq6ExMTzk+6QpOTk/nEJz5xflboAcDmMzk5mRMnTqTb7foBHgC8BDaPCti2bdv5zaMbb7wx27ZtK5yIO++8s29+29veVigJADAorVYrY2Nn3v6OjY2l1WoVTgQAo8Hm0WXa6I2e97///VlZWckv//IvZ/v27Rt6bV66D3/4w33z0aNHv+ljAMBoazab2b17d44fP549e/Z4DwYAl0l5VMi2bdsyMTHhTcuQWFlZ6ZuXl5cLJQEABqnVamV5ednWEQC8BAO9ba2qqndWVfX5qqoWq6r62TU+57+pquo/VVW1UFWVR15QxM6dO/vm8fHxQkkAgEFqNps5cuSIH+ABwEswsPKoqqrrknwoybuSfGeS91RV9Z0v+pzXJ7k3ydvqur4jyfsHlQfWMz093TcfPHiwUBIAAAAYLoPcPPreJIt1XZ+s6/obSf5tkh950ef8ZJIP1XX95SSp6/qpAeaBNT3zzDPrzgAAALBVDbI8+rYkf37B/MTZj13obyb5m1VVPVxV1X+oquqdA8wDazp06FDf/IEPfKBQEgAAABgugzwwu7rIx+qL/Ptfn+QHkrw2yWeqqnpDXddf6btQVf1Ukp9KvvlsGtgIzz777LozAAAAbFWD3Dx6Ism3XzC/Nsmpi3zOb9V1/Xxd119I8vmcKZP61HX9q3Vdv7mu6zfffPPNAwvM1tVoNNadAQAAYKsaZHn0H5O8vqqq76iq6mVJ/n6Sj73oc/7vJJNJUlXVTTlzG9vJAWaCixobG1t3BgAAgK1qYH9Druu6m2RfkhNJ/ijJR+u6Xqiq6uerqvrhs592Ikmnqqr/lGQ+yYG6rjuDygRrufXWW/vm2267rVASAAAAGC4DvTenruvjSY6/6GP/7IJ/rpP8o7O/oJinn366b37qKQ/+AwAAgGSwt63ByJiamuqb77rrrkJJAAAAYLgojyBJq9VadwYAAICtSnkESf7gD/6gb3788ccLJQEAAIDhojyCJL/4i7/YNx8+fLhQEgBgkDqdTvbv35/V1dXSUQBgZCiPIEm32113BgA2h3a7nYWFhbTb7dJRAGBkKI8gSaPRWHcGAEZfp9PJ7Oxs6rrOzMyM7SMAuEzKI0jy3ve+t2/+sR/7sUJJAIBBabfb6fV6SZJer2f7CAAuk/IIkszNzfXNv/M7v1MoCQAwKPPz8+dvTe92u9/0/R8AuDjlESRZWVnpm5eXlwslAQAGZXJy8vyt6Y1GI1NTU4UTAcBoUB5Bkp07d/bN4+PjhZIAAIPSarUyNnbm7e/Y2FharVbhRAAwGpRHkGR6erpvPnjwYKEkAMCgNJvN7N69O1VVZc+ePdm+fXvpSAAwEpRHkGTXrl259dZbkyS33XZbbr/99sKJAIBBaLVaueOOO2wdAcBL4HnkAABsGc1mM0eOHCkdA4ACjh49mqWlpQ273smTJ5MkBw4c2LBrJsnExET27t27ode8WjaPIMni4mK+9KUvJUmefPLJ838IAAAAwMVcf/31uf7660vHuCZsHkGSw4cP9833339/PvzhDxdKAwAAwEYbtm2eUaI8giQrKyt98/LycqEkAMA5G317QZKcOnUqSbJjx44Nu+Yw3l4AABvJbWuQ5BWveMW6MwCwOTz33HN57rnnSscAgJFi8wiSdLvddWcA4NobxDbPuUNNH3jggQ2/NgBsVjaPIMldd93VN7/jHe8olAQAAACGi/IIkrRarXVnAAAA2KqUR5Dkz/7sz/pmB2YDAADAGcojSHLo0KG++QMf+EChJAAAADBclEeQ5Nlnn113BgAAgK1KeQRJGo3GujPAsOt0Otm/f39WV1dLRwEAYJNRHkGSqqrWnQGG3bFjx/L444/n2LFjpaMAALDJKI8gybd8y7esO1OObQq4tE6nk/n5+STJ3NycrxcAADaU8gjizKNh1m63s7CwkHa7XToKDK1jx46l1+slSXq9nu0jAAA2lPIIkuzcubNvHh8fL5SEC3U6nczOzqau68zMzNimgDV86lOf6psfeuihMkEAANiUlEeQZHp6um8+ePBgoSRcqN1u921T2D6Ci6vret0ZAACuhvIIktx4441986tf/epCSbjQ/Px8ut1ukqTb7WZubq5wIhhOP/ADP9A3T05OlgkCAMCmpDyCnNlwGRs78+UwNjZmw2VITE5OptFoJEkajUampqYKJ4LhdM899/T9GXbPPfcUTgQAwGaiPIKc2XC58PYoGy7DodVq9f2FuNVqFU4Ew6nZbJ7fNpqamsr27dsLJwIAYDNRHkHObLhUVZUkqarKhsuQaDab2b17d6qqyp49e/yFGNZxzz335A1veIOtIwAANlzjUp9QVdVrkvxPSXbUdf2uqqq+M8lb67r+tYGng2vkXe96Vz7xiU8kOXPQ7Lvf/e7CiTin1WpleXnZ1hFcQrPZzJEjR0rHAICRdPTo0SwtLZWOsa6TJ08mSQ4cOMLe6o8AACAASURBVFA4ydomJiayd+/e0jEYgEuWR0l+Pcn/keTnzs5/kuTfJVEesWl88pOfTFVVqes6VVXl+PHj2bdvX+lYxF+IAQAYvKWlpfzpHy3l2161s3SUNTVeeFmS5PSp5wsnubgvPrNSOgIDdDnl0U11XX+0qqp7k6Su625VVS8MOBdcU/Pz8+cfbV3Xdebm5pRHAACwhXzbq3bmfW+7t3SMkfXBhw+VjsAAXU559PWqqppJ6iSpquotSZ4ZaKqrZOVwY2yllcPJycl88pOfTK/Xy9jYmDOPAAAA4KzLKY/+UZKPJZmoqurhJDcn+XsDTXWVlpaWsvRHf5ydr7qxdJQ1veyFM0/2ev7UXxROcnErz3y5dIRrqtVqnT/zqNfrOV8HgKvW6XRy6NCh3HfffQ78BwBG2iXLo7qu/5+qqv7rJH8rSZXk83VdD+dNlhfY+aob83N/Z0/pGCPrA5+dKR3hmvryl/vLsq985Sve6ANwVdrtdhYWFtJut90KDQCMtLFLfUJVVf8gSSvJm5K8Mcl7zn4MNo3Dhw/3zffff3+hJABsBp1OJ7Ozs6nrOjMzM1ldXS0dCQDgil2yPEryX13w6+1J/nmSHx5gJrjmVlb6nwywvLxcKAkAm0G73U6vd+YW9V6vl3a7XTgRAMCVu2R5VNf1+y749ZNJvifJywYfDa6dnTv7H8k5Pj5eKAkAm8H8/Hy63W6SpNvtZm5urnAiAIArdzmbRy92OsnrNzoIlPTip6u94x3vKJQEgM1gcnIyjcaZoyUbjYaneAIAI+1yzjz67aqqPnb218eTfD7Jbw0+Glw7/+bf/Ju++Td+4zcKJQFgM2i1WhkbO/M2a2xszFM8AYCRdsmnrSU5csE/d5Ms13X9xIDyQBHnbi1YawaAl6LZbGb37t05fvx49uzZ4wmeAMBIu2R5VNf1p65FECip0Wj0FUbnbjUAgCvVarWyvLxs6wgAGHlr3rZWVdXXqqr66kV+fa2qqq9ey5AwaO9973v75h/7sR8rlASAzaLZbObIkSO2jgCAkbdmeVTX9Svruv7Wi/x6ZV3X33otQ8KgvfgpOL/zO79TKAkAAAAMl8u+N6eqqluSXH9urut6ZSCJoICVlf7/d15eXi6UBAAAAIbL5Txt7YerqvrTJF9I8qkkf5bkkwPOBdfUzp07++bx8fFCSQAAAGC4XLI8SvIvk7wlyZ/Udf0dSe5K8vBAU8E1Nj093TcfPHiwUBIANotOp5P9+/dndXW1dBQAgKtyOeXR83Vdd5KMVVU1Vtf1fJL/csC54Jp67LHH+ubf//3fL5QEgM2i3W5nYWEh7Xa7dBQAgKtyOeXRV6qqekWSzyT5zaqq/nWS7iX+d2Ck/Pqv/3rf/Gu/9mtlggCwKXQ6nczOzqau68zMzNg+AgBG2uUcmP3pJK9O8j8meW+SVyX5+UGGulqnTp3K6We+mg98dqZ0lJG1/MyXc0NeKB0DAEZSu91Or9dLkvR6vbTb7ezbt69wKgCAK3M5m0dVkhNJHkryiiT/7uxtbAAAXMT8/Hy63TOL2t1uN3Nzc4UTAQBcuUtuHtV1/S+S/Iuqqr47yX+b5FNVVT1R1/U7Bp7uCu3YsSPP57r83N/ZUzrKyPrAZ2eybcdrSse4Zn78x3+879a1n/iJnygXBoCRNzk5mRMnTqTb7abRaGRqaqp0JACAK3Y5t62d81SSLyXpJLllMHHg8h09ejRLS0sDufbnPve5fO5zn9uQa01MTGTv3r0bci0ARkOr1crs7GySZGxsLK1Wq3AiAIArd8nb1qqq+u+qqnooye8muSnJT9Z1/d2DDgbXWqNxpku95RbdKABXp9lsZvfu3amqKnv27Mn27dtLRwIAuGKXs3k0nuT9dV3/v4MOAy/FRm/zHDhwIEnywAMPbOh1AdiaWq1WlpeXbR0BACPvcs48+tlrEQQAYDNpNps5cuRI6RgAAFftcp62BgAAAMAWpTwCAAAAYE3KIwAAAADWpDwCAAAAYE3KIwAAAADWpDwCAAAAYE3KIwAAAADWpDwChtpv//Zv553vfGeOHz9eOgoAAMCWpDwChtqv/MqvJEk++MEPFk4CAACwNSmPgKH127/926nrOklS17XtIwAAgAKUR8DQOrd1dI7tIwAAgGuvUTrAoKw88+V84LMzpWOs6S++/rUkyWte/srCSS5u5ZkvZ2LHa0rHYIs7t3W01gwAAMDgbcryaGJionSES/rGya8nSbYNaUEzseM1I/Hfkc2tqqq+wqiqqoJpAAAAtqZNWR7t3bu3dIRLOnDgQJLkgQceKJwEhtfP/MzP5EMf+tD5+X3ve1/BNAAAAFuTM4+AofV3/+7f7Zvf/e53F0oCAACwdW3KzSNgc1hcXOybT548mdtvv71QGoCXptPp5NChQ7nvvvuyffv20nEAWMepU6fy9WdO54MPHyodZWR98ZnlvDw3lI7BgNg8AobW4cOH++b777+/UBKAl67dbmdhYSHtdrt0FACAq2LzCBhaKysrffPy8nKhJAAvTafTyezsbOq6zszMTFqtlu0jgCG2Y8eOnM7zed/b7i0dZWR98OFDuWHHttIxGBCbR8DQ2rlzZ988Pj5eKAnAS9Nut9Pr9ZIkvV7P9hEAMNKUR8DQmp6e7psPHjxYKAnASzM/P59ut5sk6Xa7mZubK5wIAODKKY+AobVr167z20fj4+MOywZGxlvf+ta++c477yyUBADg6imPgKE2PT2dG264wdYRAABAIQ7MBobarl278uCDD5aOAfCSPProo33zI488UigJAMDVs3kEALDBJicnc9111yVJrrvuukxNTRVOBABw5ZRHALAJLC4u5u67787JkydLRyFJq9XK2NiZt1ljY2NptVqFEwEAXDnlEQBsAocPH87p06dz//33l45Ckmazmdtuuy1JsmPHjmzfvr1wIgCAK6c8AoARt7i4mJWVlSTJ8vKy7aMh0Ol08uSTTyZJnnzyyayurhZOBABw5ZRHADDiDh8+3DfbPiqv3W6nruskSa/XS7vdLpwIAODKKY+AodbpdLJ//34/tYd1nNs6Omd5eblQEs6Zn59Pt9tNknS73czNzRVOBABw5QZaHlVV9c6qqj5fVdViVVU/u87n/b2qquqqqt48yDzA6Gm321lYWPBTe1jHzp07++bx8fFCSThncnIyjUYjSdJoNDxtDQAYaQMrj6qqui7Jh5K8K8l3JnlPVVXfeZHPe2WS/yHJ7w0qCzCaOp1OZmdnU9d1ZmZmbB/BGqanp/vmgwcPFkrCOZ62BgBsJoPcPPreJIt1XZ+s6/obSf5tkh+5yOf9yySHkzw3wCzACGq32+n1ekmcGQLr2bVr1/nto/Hx8dx+++2FE9FsNrN79+5UVZU9e/Z42hoAMNIGWR59W5I/v2B+4uzHzquq6nuSfHtd1x8fYA5gRDkzBC7f9PR0brjhBltHQ6TVauWOO+6wdQQAjLxBlkfVRT5Wn//NqhpL8r8k+ceXvFBV/VRVVY9VVfXY008/vYERgWHmzBC4fLt27cqDDz5o62iINJvNHDlyxNYRADDyBlkePZHk2y+YX5vk1AXzK5O8IclDVVX9WZK3JPnYxQ7Nruv6V+u6fnNd12+++eabBxgZGCatVitVdaaHdmYIAABAGYMsj/5jktdXVfUdVVW9LMnfT/Kxc79Z1/UzdV3fVNf16+q6fl2S/5Dkh+u6fmyAmYAR0mw2c9tttyVJbrvtNj+9BwAAKGBg5VFd190k+5KcSPJHST5a1/VCVVU/X1XVDw/q3wtsHp1OJ08++WSS5NSpU562BgAAUMAgN49S1/Xxuq7/Zl3XE3Vdf+Dsx/5ZXdcfu8jn/oCtI+BC7XY7dX3mqLS6rj1tDQAAoICBlkcAV8PT1gAAAMpTHgFD661vfWvffOeddxZKAgAAsHUpjwAAAABYk/IIGFqPPvpo3/zII48USgIAALB1KY+AofXGN76xb37Tm95UKAkAAMDWpTwChtYXvvCFvvnkyZOFkgAAAGxdyiNgaH3xi19cdwYAAGDwlEfA0LrxxhvXnQEAABg85REwtL785S+vOwMAADB4yiMA2AQ6nU7279+f1dXV0lEAANhklEfA0Kqqat0Z+GvtdjsLCwtpt9ulo3DW4uJi7r77bof9AwAjr1E6AMBafuZnfiYf+tCHzs/ve9/7CqaB4dXpdDI7O5u6rjMzM5NWq5Xt27eXjrXlHT58OKdPn87999+fD3/4w6XjXBNHjx7N0tJS6RjrOlfmHThwoHCStU1MTGTv3r2lYwDAecojYGjdeeedfeXRW97yloJpYHi12+30er0kSa/XS7vdzr59+wqn2toWFxezsrKSJFleXs7Jkydz++23F041eEtLS/nDz38+1fabSkdZU12f+Z+PP90pG2QN9epflo4AAN9EeQQMrRfffuMvxHBx8/Pz6Xa7SZJut5u5uTlfK4UdPny4b95K20fV9puy7Yd+pHSMkfX8x3+rdAQA+CbKI2Bozc/P983+QgwXNzk5mRMnTqTb7abRaGRqaqp0pC3v3NbROcvLy4WSAHC5vvjMSj748KHSMdb0l1//iyTJTS9/TeEkF/fFZ1by+h0TpWMwIMojYGi98Y1vzGc/+9nz85ve9KaCaWB4tVqtzM7OJknGxsbSarUKJ2Lnzp19BdL4+HjBNABcysTE8Jce3ZPfSJLcsGNb4SQX9/odEyPx35ErozwChtYXvvCFvtkTi+Dims1mvvd7vzef/exn833f930Oyx4C09PTfZuSBw8eLJgGgEsZhUPqzx30/8ADDxROwlY0VjoAwFq++MUvrjsDf+1c2apkHQ67du1Ko3HmZ3SNRmNLHJYNAGxeyiNgaN16661982233VYoCQy3xcXF8+XqF7/4RQXSEFhcXOw7xNxrAgCMMuURMDLqc89XBvpc7MlelOU1AQA2E+URMLS+9KUvrTsDZ3iy1/DxmgAAm4nyCBhaO3fu7Js9rQguztfK8PGaAACbiaetXaajR49maWlpw663uLiYv/qrv8r73//+bNu2MY9anJiYGImnBMDl8rQiuDy+VoaP1wQA2ExsHhXS6/XS6/Xy1FNPlY4CQ2vXrl3nf3o/Pj7uaUWwBl8rw8drAgBsJjaPLtNGbvR0Op38w3/4D5Mkzz77bO69995s3759w64Pm8n09HSmp6f91B4uwdfK8PGaAACbhc2jAtrtdnq9XpIzG0jtdrtwIhheN954Y26//fa8+tWvLh0FhtquXbvy4IMP2nABAGDDKY8KmJ+fT7fbTZJ0u93Mzc0VTgTDq91uZ2FhQckKjJzDhw/n9OnTuf/++0tHAQC4KsqjAiYnJ9NonLljsNFoZGpqqnAiGE6dTiezs7Op6zozMzNZXV0tHQngsiwuLmZlZSVJsry8nJMnTxZOBABw5ZRHBbRarYyNnflPPzY2llarVTgRDCe3eAKj6vDhw32z7SMAYJQpjwpoNpvZvXt3qqrKnj17HJYNa3CLJzCqzm0dnbO8vFwoCQDA1VMeFdJqtXLHHXfYOoJ1uMUTGFU7d+7sm8fHxwslAQC4eo3SAbaqZrOZI0eOlI4BG+ro0aNZWlrasOs9//zz5zePXnjhhSwuLubAgQNXfd2JiYns3bv3qq8DsJbp6ens27fv/Hzw4MGCaQAAro7NI2Bobdu27fzm0Y033pht27YVTgRweXbt2nV++2h8fDy333574UQAAFfO5hGwYQaxzfP+978/Kysr+eVf/mXngwEjZXp6OtPT07aOAICRpzwChtq2bdsyMTGhOAJGzq5du/Lggw+WjgEAcNXctgYAm0Cn08n+/fuzurpaOgoAAJuM8qgQb/IB2EjtdjsLCwtpt9ulowAAsMkojwrxJh+AjdLpdDI7O5u6rjMzM+MHEwAAbCjlUQHe5AOwkdrtdnq9XpKk1+v5wQQAABvKgdkFXOxN/r59+wqnGryjR49maWmpdIw1nTx5Mkly4MCBwknWNzExMZCnmsHl6nQ6OXToUO677z4HmQ+J+fn5dLvdJEm3283c3NyW+L4CAMC1oTwqYKu+yV9aWsriH/1hXvutVekoF7WtWydJnvvi44WTrO2Jr9alI0COHTuWxx9/PMeOHcv+/ftLxyHJ5ORkTpw4kW63m0ajkampqdKRAADYRJRHBWzlN/mv/dYq73/ry0rHGFn/6tFvlI7AFtfpdDI/P58kmZubyz333GP7aAi0Wq3Mzs4mScbGxtJqtQonAgBgM3HmUQGtVitjY2f+03uTD4ySY8eO9d12e+zYscKJSJJms5ndu3enqqrs2bNHoQcAwIZSHhXQbDbz9re/PUny/d///d7kAyPjoYce6pvPbSFRXqvVyh133OEHEgAAbDi3rQFw2aqqWnemnGazmSNHjpSOAQDAJmTzqIBOp5PPfOYzSZJPf/rTWV1dLZwI4PK85S1v6ZvvvPPOQkkAAIBrRXlUQLvd7jszpN1uF04EcHm+5Vu+pW9+2cscgA8AAJud8qiA+fn5dLvdJEm3283c3FzhRACX55FHHumbH3744UJJYPh1Op3s37/fhjEAMPKURwVMTk6m0Thz3FSj0cjU1FThRACX5+abb+6bb7nllkJJYPi12+0sLCzYMAYARp7yqIBWq5WxsTP/6cfGxjwZBxgZTz/9dN/81FNPFUoCw63T6WR2djZ1XWdmZsb2EQAw0pRHBTSbzezevTtVVWXPnj3Zvn176UgAl+XFB2S/7W1vK5QEhpvzDQGAzaRROsBW1Wq1sry8bOsIADahi51vuG/fvsKpBu/UqVOpv/q1PP/x3yodZWTVnb/Mqef/qnQMAOhj86iQZrOZI0eO2DoCRsqjjz7aN7/4AG3KcTjzcJmcnOybnW8IAIwym0eFdDqdHDp0KPfdd58CCRgZk5OT+ff//t/nhRdeyHXXXecvxEPk2LFjefzxx3Ps2LHs37+/dJwt7w1veEM+8YlPnJ+/+7u/u2Caa2fHjh1Z3dbJth/6kdJRRtbzH/+t7Li5WToGAPSxeVSIJ7AAo6jVauW6665Lklx33XVuvR0SnU4n8/PzSZK5uTnbR0PgQx/6UN/8S7/0S4WSAABcPeVRAZ7AAowqB/4Pp2PHjvUdznzs2LHCiXj22WfXnQEARonyqABPYAFGWavVyh133GHraIh86lOf6psfeuihMkE47+Uvf/m6MwDAKHHmUQFb9QksQBlHjx7N0tLShl3v1KlTSZJDhw5t2DUnJiayd+/eDbveVlPX9boz194dd9yRz33uc+fnN7zhDQXTAABcHZtHBUxOTqbRONPbNRoNB84CI+W5557Lc889VzoGF7j11lv75ttuu61QEs55/PHH++Y//MM/LJQEAODq2TwqoNVqZXZ2NkkyNjbm1g9goDZ6o+fAgQNJkgceeGBDr8uVe/HZeZ1Op1ASzpmcnOx72pofFAEAo8zmUQEOnAVgI724mLjrrrsKJeGcVquVsbEzb7P8oAgAGHXKo0IcOAvARmm1Wtm2bVuS5GUve5nvLQAAbCjlUSHNZjNHjhyxdQTAVWs2m9mzZ4+N1iHSbrf7No88WRUAGGXKIwDYBN71rnflb/yNv5F3v/vdpaOQiz9ZFQBgVDkwGwA2gU9+8pP5z//5P+f48ePZt29f6Tgj6ejRo1laWtqQa11//fU5ffp033zusPmrMTExseGH4AMAXIrNIwAYcZ1OJ7Ozs6nrOjMzM9/09DWuvVtuueX8P1dV1TcDAIwam0dcM6dOncrpr9b5V49+o3SUkfXEV+vcUJ0qHQMYMu12O71eL0nS6/XSbrdtH12Bjd7oabVaWV1dzQ/+4A96Pf7/9u49TI6qzv/455MMOEYwSi7CcJMMeAG8JvoDlEsiILAqolFxdDVeHjfrKuuFoLu6iO6zrhr96U8UIyBEuYiKKBAigYVw0YSFEJAkQJAZEoUAkgQCESEkOb8/zumkp9PV0zPpTlX3vF/PkydVNVWnv12nqk7Vt05VAwCAlkbPIwAAWhzv1ymm8ePHa9SoUfz6HQAAaHn0PMIO09XVpWfCWn3m0J3zDqVlfW/hBnV2deUdBoCCmTx5subNm6eNGzeqo6NDU6ZMyTskSNppp53U3d3Nr98BAICWR88jAABaXE9PjzZt2iRJ2rRpEz1dAAAA0FAkjwAAaAMhhH7/AwAAAI1C8ggAgBZ33nnn1RwHAAAAtgfJIwAAWtwNN9zQb3z+/Pn5BAIAAIC2RPIIAIAWZ7vmOAAAALA9SB4BANDiDjnkkH7jhx12WE6RAAAAoB2RPAIAoM3w0mwAAAA0UkfeAQAAgO2zcOHCfuMLFizIKRJACmtX67k5l+cdRqbw5DpJkl84OudIqgtrV0vjxuQdBgAA/ZA8AgAAQEN0d3fnHcKA+p6KyaMJRU3QjBvTEusRADC8kDwCAKDF7b777nrooYf6jQN5mD59et4hDGjGjBmSpJkzZ+YcCYB2N2vWLPX29jasvL6+Pklbj2ON0N3d3RLHbuSP5BEwjDW6QWuGZjSSjUaji7ytXbu25jgAAGh9nZ2deYeAYYzkETCM9fb2avm9SzT2RXlHUsPm+N+aR5bkG0eG1U/kHQEgTZkyRVddddWW8be85S05RgMAAKTW6I0J1IvkETDMjX2R9K7JzjuMlnXZfH7VCvk7/vjj+yWPTjjhhByjAQAAQLsZkXcAAABg+/zud7/rNz537tycIgEAAEA7InkEAECLu/766/uNX3fddTlFAgAAgHZE8ggAgBY3bty4fuPjx4/PKRIAAAC0I5JHAAC0uEcffbTf+COPPJJTJAAAAGhHJI8AAGhxO+20U81xAAAAYHuQPAIAoMWtX7++5jgAAACwPUgeAQDQ4vbZZ59+4/vuu29OkQAAAKAdkTwCAKDFnXbaaf3Gv/CFL+QUCQAAANpRU5NHto+zvdz2/ba/WOXvn7N9t+27bF9nm1ulAAAM0v7777+l99G+++6rCRMm5BwRAAAA2knTkke2R0r6oaTjJR0o6f22D6yY7Q5Jk0IIr5Z0qaRvNSseAADa2WmnnaZRo0bR6wgAAAAN19HEst8o6f4QQp8k2b5E0omS7i7NEEKYXzb/LZI+2MR4AABoW/vvv78uu+yyvMMAAABAG2rmY2t7SvpL2fiDaVqWj0n6XRPjAQAAAAAAwCA1s+eRq0wLVWe0PyhpkqQjM/7+CUmfkLb9RRm0lgefDPrewg15h1HVY3+Lm+e4F1TbdIvhwSeD9q+VggXQEmbNmqXe3t6Glrlq1SpJUldXV8PK7O7u1vTp0xtWHgAAAFpTM5NHD0rau2x8L0mrKmeyfbSkL0k6MoTwbLWCQghnSzpbkiZNmlQ1AYXi6+7uzjuEmp7r65Mkde5Z3BfN7r9n8dcjgHw888wzeYcAAACANtXM5NFtkg6wvZ+khySdLKmnfAbbr5P0Y0nHhRD+2sRYUABFv3s9Y8YMSdLMmTNzjgRAu2vG8ZBjGAAAAJqlae88CiFslPQpSfMk3SPplyGEZba/ZvsdabaZknaR9Cvbd9q+olnxAAAAAAAAYPCa2fNIIYS5kuZWTDu9bPjoZn4+AAAAAAAAtk8zf20NAAAAAAAALa6pPY8AFNuqVav01JPSZfN5D/1QrX5CenbzNr8FAAAAAABtg55HAAAAAAAAyETPI2AY6+rq0poRa/Suyc47lJZ12fygMbt35R0GAAAAADQNPY8AAAAAAACQieQRAAAAAAAAMpE8AgAAAAAAQCaSRwAAAAAAAMhE8ggAAAAAAACZSB4BAAAAAAAgE8kjAAAAAAAAZCJ5BAAAAAAAgEwkjwAAAAAAAJCpI+8AAORr9RPSZfND3mFkWrc+/j96l3zjyLL6CWnM7nlHAQAAAADNQ/IIGMa6u7vzDmFA6/r6JEljdp+QcyTVjdm9NdYjAAAAAAwVySNgGJs+fXreIQxoxowZkqSZM2fmHAkAAAAADE+88wgAAAAAAACZSB4BAAAAAAAgE8kjAAAAAAAAZCJ5BAAAAAAAgEwkjwAAAAAAAJCJX1sDAAAtZ9asWert7c07jJr6+vokbf3VyKLq7u5uiV/fBAAA+SF5BAAAWk5vb6+WLL9XGjM671CyhU2SpCWrH845kBrWrMs7AgAA0AJIHgEAMAB6uTRGw3u4jBmtkW87vHHlDUOb5tycdwgAAKAFkDwCAGAAsZfLUmnsznmHUsNGSdKSNfflHEeG1RvyjgAAAABDRPIIAIB6jN1ZPnGPvKNoWeHyAj+6BQAAgJr4tTUAAAAAAABkInkEAAAAAACATCSPAAAAAAAAkIl3HgFAwRT9l71a4Ve9pCb8shcAAAAwTJE8AoCC6e3t1T333qVdd8s7kuo2hvj/g3+9K99Aanhqbd4RAAAAAO2D5BEAFNCuu0lvfGveUbSuW+flHQEAAADQPnjnEQAAAAAAADKRPAIAAAAAAEAmkkcAAAAAAADIxDuPAAAYwKpVq6SnNihc/nDeobSu1Ru06tlVeUcBAACAIaDnEQAAAAAAADLR8wgAgAF0dXVpzZr18ol75B1KywqXP6yuMV15hwEAAIAhoOcRAAAAAAAAMpE8AgAAAAAAQCYeWwMAAC0nvsR8nTbNuTnvUFrbmie0akPIOwoAAFBw9DwCAAAAAABAJnoeAQCAltPV1aU1q62Rbzs871Ba2qY5N6trLC+CBwAAtdHzCAAAAAAAAJlIHgEAAAAAACATj60BAFCP1RsULn847yiyrdsY/x9d0KZ99QZpTN5BAAAAYCgKeoYJAEBxdHd35x3CgPrW9UmSJoyZkHMkGca0xnoEAADAtkgeAQAwgOnTp+cdwoBmzJghSZo5c2bOkQAAAKDd8M4jAAAAAAAAZCJ5BAAAAAAAgEwkjwAAAAAAAJCJ5BEAAAAAAAAykTwCAAAAAABAJpJHAAAAAAAAyETyCAAAAAAAAJlIHgEAAAAAACATySMAAAAAAABk6sg7e3pAFAAAHPVJREFUAGCoZs2apd7e3oaV19fXJ0maMWNGw8qUpO7ubk2fPr2hZQIAAAAAsKOQPAKSzs7OvEMAAAAAAKBwSB6hZdGbBwCGuTXrtGnOzXlHkW3d+vj/6F3yjaOWNeuksXvkHQUAACg4kkcAGqbRjxJK0vLly7VhwwZ9/OMf14tf/OKGlMmjhEDr6+7uzjuEAfU9GR+HnlDk5MzYPVpiXQIAgHyRPAJQaBs2bJAkPfjggw1LHhXdqlWr9NST0q3z8o6kdT21Vlq1cVXeYaCJWiEBXHqH3syZM3OOBAAAYPuQPALQMI2+mJs/f76WLFmyZfztb3+7jjjiiIZ+BgAAAACgNpJHAArrO9/5Tr/xb33rW8MiedTV1aXNHav1xrfmHUnrunWe1DW+K+8wAAAAgLYwIu8AACDLxo0ba44DAAAAAJqP5BGAwho5cmTNcQAAAABA85E8AlBYHR0dNccBAAAAAM1H8ghAYT377LM1xwEAAAAAzUfyCEBhjRo1quY4AAAAAKD5SB4BKKyDDz643/irXvWqnCIBAAAAgOGL5BGAwlq6dGm/8SVLluQUCQAAAAAMXySPABTWoYce2m/8sMMOyykSAAAAABi++OkiAAAAFNKsWbPU29vb0DL7+vokSTNmzGhYmd3d3Zo+fXrDygMAoGhIHgEorIULF/YbX7BgQU6RAADaRWdnZ94hAADQckgeASisyZMn6+qrr9amTZs0cuRITZkyJe+QAAA7EL15AAAoBpJHAAqrp6dH11577ZbkUU9PT94hAQ3BozgAAABoJbwwG0BhjRkzRsccc4xs69hjj9Vuu+2Wd0hAYXV2dvI4DgAAAJqCnkcACq2np0crV66k1xHaCr15AAAA0EroeQSg0BYsWKClS5fqlltuyTsUAAAAABiWSB4BKLSzzjpLknTmmWfmHAkAoB2sWbNGp556qtauXZt3KAAAtAweWwNQWFdeeaVCCJKkEILmzp2rE044Ieeodoyn1kq3zss7iuqefir+P2rXfOOo5am1ksbnHQWAIrr44ou1bNkyXXzxxfrUpz6VdzgAALQEkkcACqvU66jkzDPPHBbJo+7u7rxDqKlvffxVr73GT8g5khrGF389Atjx1qxZo2uvvVYhBF1zzTXq6enhxxgAAKgDySMAhVXqdZQ13q6K/jLl0k/Bz5w5M+dIAGBwLr74Ym3evFmStHnzZnofAQBQJ955BKCwbNccBwBgMObPn6+NGzdKkjZu3Kjrr78+54gAAGgNJI8AFNaBBx7Yb/yggw7KKRIAQDuYPHmyOjpix/uOjg5NmTIl54gAAGgNJI8AFNYDDzzQb7yvry+nSAAA7aCnp0cjRsTT3xEjRqinpyfniAAAaA0kjwAU1utf//p+4xMnTswpEgBAOxgzZoyOOeYY2daxxx7Ly7IBAKgTL8wGUFj0PAIANFpPT49WrlxJryMAAAbBrfbrRZMmTQqLFi3KOwwAO8Bxxx23zbSrr746h0ha26xZs9Tb29uw8kpJvAkTJjSszO7u7sL/yhzaH/sKAAAYzmzfHkKYVO1vPLYGoLD23HPPmuPIR2dnpzo7O/MOAyg89hUAANAumvrYmu3jJP0/SSMlnRtC+EbF358n6WeSJkpaI+l9IYQVzYwJQOvYb7/99NBDD20Zb+Td++GEXgpAfdhXAAAAqmtazyPbIyX9UNLxkg6U9H7bB1bM9jFJj4cQ9pf0XUnfbFY8AFrP4sWL+43ffvvtOUUCAAAAAMNXMx9be6Ok+0MIfSGEDZIukXRixTwnSvppGr5U0ltsu4kxAWghe+21V7/xvffeO6dIAAAAAGD4ambyaE9JfykbfzBNqzpPCGGjpHWSxjQxJgAt5L777us3vnz58pwiAQAAAIDhq5nJo2o9iCp/2q2eeWT7E7YX2V702GOPNSQ4AAAAAAAADKyZyaMHJZU/Y7KXpFVZ89jukDRa0trKgkIIZ4cQJoUQJo0bN65J4QIAAAAAAKBSM5NHt0k6wPZ+tneWdLKkKyrmuULSh9PwVEnXhxC26XkEYHiaNm1av/GPfexj+QQCAAAAAMNY05JH6R1Gn5I0T9I9kn4ZQlhm+2u235Fm+4mkMbbvl/Q5SV9sVjwAWs/JJ5/cb/w973lPTpEAAAAAwPDV0czCQwhzJc2tmHZ62fAzkrgaBJBp2rRpmj17Nr2OAAAAACAnbrWnxCZNmhQWLVqUdxgAAAAAAABtw/btIYRJ1f7WzHceAQAAAAAAoMWRPAIAAAAAAEAmkkcAAAAAAADIRPIIAAAAAAAAmUgeAQAAAAAAIBPJIwAAAAAAAGQieQQAAAAAAIBMJI8AAAAAAACQieQRAAAAAAAAMpE8AgAAAAAAQCaSRwAAAAAAAMhE8ggAAAAAAACZSB4BAAAAAAAgE8kjAAAAAAAAZCJ5BAAAAAAAgEwkjwAAAAAAAJCJ5BEAAAAAAAAykTwCAAAAAABAJpJHAAAAAAAAyOQQQt4xDIrtxyStzDuOBhkraXXeQaAf6qSYqJfioU6KiXopHuqkmKiX4qFOioc6KSbqpXjaqU72DSGMq/aHlksetRPbi0IIk/KOA1tRJ8VEvRQPdVJM1EvxUCfFRL0UD3VSPNRJMVEvxTNc6oTH1gAAAAAAAJCJ5BEAAAAAAAAykTzK19l5B4BtUCfFRL0UD3VSTNRL8VAnxUS9FA91UjzUSTFRL8UzLOqEdx4BAAAAAAAgEz2PAAAAAAAAkInkkSTbp9i+x/ZFtt9h+4sNKnd9A8rIjKdUvu0u25em4dfaPmF7P7eobL/U9tI65ukpG59k+/tpeJrtHzQxvq/ZPrrK9KNsz0nDW+rU9jttH9isePJke7rtD6Xhaba7asxbdb01Oo6K6QNuS+2kWevB9g222/7XJZrN9mzbU6tMH3T9lLcJVf7W0vVV3l7nHEf5Mf15tv/H9p2239eg8rdsD7bPHWo7YXvBQOVvR4wvsv3JOubr1yYPMN92H5Ntn2H71DT8ilQvd9ju3t6yU5krbI9Nw1XXbx1lbDkvqVV+qyqvgwaVNzdtb3Vtc2icyjZjuJ077Qj1XJsM5frF9mdsj9q+6Ia3oR6PB9vGttJ+1ZF3AAXxSUnHhxAeSONX5BlMuRDCFRognhDCKkmlDfS1kiZJmtvk0IrspZJ6JF0sSSGERZIW7YgPDiGcXsc85XX6TklzJN3dzLjyEEKYVTY6TdJSSasq57M9sp711qA42oJtKz52vLneZdpxPZTY7gghbMw7jiJI66K8TWg3le21pNy3gddJ2imE8Np6FxhMvCGEjw81sBDCYUNdtg4vUqyPswaY76Uqa5N3sHdKujyE8JV6Fxhk3Qxp/e7I85J2EEI4QYoXWKpvmwMgfUbShZKezjsQtI9h3/PI9ixJEyRdYfuz5Zld25eX9Zz4p9KdTtvdtq+2fbvtm22/Ik3fz/ZC27fZ/s8an/nbtOwy258om36c7cW2/2j7ujStPJ6q5ZeylbZ3lvQ1Se8r3QG1/Sfb49J8I2zfX6Q7Wra/WX4XKd2t+ryjmel7Lal2Nzd975vTOltsu3QS9w1Jh6d18NnyO8QVy4+z/eu0Pm+z/aZBfIZsn5Zi+6Ptb6Rp5XeLj7N9r+3fS3pX2XLTbP8glfUOSTNTrN22F5fNd4Dt24ewWnc42x+yfVdaFxekaWfYPjWtj0mSLkrf8/kpk396WjfvqVhvb7C9IJV1q+1dKz5rF9vXpfpYYvvEeuJIwxPT3xZK+pcds3YaI22L99g+S9JiSXvbPjYdExbb/pXtXdK837B9d1oX307TBlwPrrizZXuO7aPS8I9sL0rHra/WEW+1GPrdifHW3pMjbJ+Vyp7jeJe5tD2cnvbPpbbPtu00/QbbX7d9o6R/3a6Vm4Nq26qkI9K23+fqvZA6bZ+ftvs7bE9O06el+r9S0jUuu4OV9rdL0mf9QtLzy8qre/spAm/bXp+RtolrJP3M9kjHduO2FPs/lS07o2z6NttvWna2t7Y5n03Tt9x1tz3W9oqK5cYrnpy/1luP4+U9UybZviEN94u3ohw7tgt3275K0viyv5XH8P4U31Lb30zT9nVs68emfelm28emv62vo/yJtm90PC+ZZ3uPOqvkG5K60/eemT6jWrtd2SZntqvV2N7D9k1p+aW2Dy//bml4qu3ZFcudoHjx9HHb811xZ9exfTqjbB1XPZ7YHmP7mrTP/ViSy/5Wvn63+e62T3Lsleb0Pe6zvbv791yrVf4HHdvBO23/2PbIeiomD7a/ZHu57f+R9PI0Let8ebbt77vieFejrkv7VOU2d4H7nwNcZPsdO/zLt4G0f9xr+6eOx8lLTa+VIbP9AttXObbxSx2vyaq2DRXLzbY9K+0v99l+W9mfu9L+9Cfb3ypbZpvzM9unSOqSNN/2/DStpdr8Hc0Z1+dlf692jbGv4zXJXen/fcoW2eacLqutaCkhhGH/T9IKSWPT8DRJP0jDL5F0v6TDJd0nabc0/TpJB6Th/yPp+jR8haQPpeF/kbQ+4/NK5TxfsTfGGEnjJP1F0n4V85THU7V8xbt6SyvnT+NfkfSZNHyspF/nvb4r1sXrJN1YNn63pH0kvVvStZJGpnr4s6Q9Kr7rKEmdafgASYvS8FGS5pSVuWW8Yn1eLOnNaXgfSfdUiS/rM46XtEDSqIr6mq14x78z1ecBiieCv8yIYbakqWWfN1/Sa9Pw1yV9Ou86qqMOD5K0XFv3odK6OEPSqWn4BkmTKva508rGS+ttZ0l9kt6Qpr9QUkfF53VIemEaHqu4j7rOOO6SdGQanlnallrhX9r2N0s6pOy73yTpBWn8C5JOl7RbWg+lH0R4Ub3rQdseP+ZIOqpifY5M9fnqanVbmjcjhsrtvXQMm6rYW3KEpN0lPV6ar/S5afgCSW8v+9yz8q6XIdblNttqWje/SuvgQEn3l9V7qX4+L+n8NPwKxeNiZ6q3B8vqqHyZz0k6Lw2/WtJGxWTuoLafovxT//b6DEm3S3p+Gv+EpC+n4ecp9uzYT7HtO1vxODEibddHVJQ7UdK1ZeOlbXbL9p3W2Yo0fJS2HtO3DFeJcZKkG6rFW/H579LWNq9L0hNl+8ANqZyuVOfjFI+D10t6Z5rn45IulTRD0o+r7GNVy5e0k2JbNi7N977S9lJHXWzZztJ4VrtduX6y2tV+5ZXN/3lJX0rDIyXtWv7d0vBUSbPL1vOpVYYr4z1V0hll67jq8UTS9yWdnob/QVIoq9/1tb57+tuFkj6luN29v8r2U7V8Sa+UdKVirzYp9rb5UN77YMY6mihpSarbFyq2y6cq+3x5tqof77LqekVaJ5V1eKSk36bh0ZIeUMU5A//qrsOXpm3vTWn8vFSHNyi2CXemf3dX20/5t836fLekc8rGRyu7bZim/tcFV6d94wDFtr3UzvelcjolrZS0d1om6/ys/PNass3fwXVW7fq8dOzJusa4UtKH0/BHy45HWce4Aa9vi/6Px9ZqCCE8avt0xQv6k0IIa1OW9jBJv7K33Bx6Xvr/TYobhRQvcr6ZUfQptk9Kw3srHhzGSboppK74IYS1VZart/xy50m6XNL3FDfq8+tYZocJIdxhe7zj+3DGSXo8hPBnx7u+Pw8hbJL0qOPdwDcoXvSW7CTpB7ZfK2mTpJcN8uOPlnRgWT2+0PauIYSn6viMoxUv4p5O36Oyvl4h6YEQwp8kyfaFihc2AzlX0kdsf07xJP6Ng/xOeZgi6dIQwmopc9ut5hdVpr1c0sMhhNtSWU9WmceSvm77CMVkyp6KB+Cacdgerdgo3pgmXaCYBGwlK0MIt6ThQxQbpD+kbXhnSQslPSnpGUnnOvYw6NfrbjvWw3vTnZgOxYbuQPXfH8vVjKGKN0v6VYiP4T1SukuWTLZ9muJFyW6Slik21lL1bagVbLOtpjr8bVoHd9t+SZXl3izpzLTMvbZXausx6dqMfe8IxYtThRDusl2qsyFtPwV0RQjh72n4WEmv9tZeW6MV29dj07870vRd0vSbysrpkzTB9pmSrpJ0zQ6It9wR2trmrbJ9fZV53qB4sfGYFHtZpOV+G0I41/Z7JE1XfHy93vJfLulgSdem7WCkpIeH+N3erOrtduVxfLBt922SzrO9k+J3vXOI8Q0k63hyhFLv4RDCVbYfrzJP1ne/QtKnFS9Ebgkh/HwQ5b9FMSlzW6qb50v661C+2A5wuKTflM6JbF+heIGbdb4sVT/eDaquQwg32v6hYw/AdyneIOUR5qH7SwjhD2n4QkmnpOEPhPiopRwfHSx6u1AESyR927GH6JwQws1l+8FAfpn2jT/Z7lO8ppCk60II6yTJ9t2S9lW8UV3P+Vm7tPnNVO36vCTrGuNQbX265AJJ3ypbptoxLqutyDqfLhySRwN7laQ1infqpJhBfCJkv9sg1CrM8RGQoyUdGkJ4OnVZ7FS8IK65bD3lbzNzCH+x/ajtKYp3fT4wmOV3kEsV7xruLumSNK2eI+xnJT0q6TWK9fLMID93hGI9VDuRH+gz6qmvQdVV8mvF3mLXS7o9hLBmCGXsaPVuu5X+NsSyPqCYaJwYQnjO8TGSevahocZZJOXrzIoJg/dXzmT7jYoXHicr3vGeUrFc1nrYqP6PM3em8vZTvAP5hhDC446Ph3RmBRlC2JgRw5byHc9edi6LaRu2OxXvtk9Kx7IzKj632jbUCrLq4NmKeaotl6XWuqj2WUPdfoqmcp/4dAhhXvkMtt8q6b9DCD/OKiRt16+R9FbFnr3vVbzhUr5PZG7zFWotM9h6KpdZ/46Pl+yVRneR9FSV2bK2g2UhhEMH+Ox61HtlNKi2O4RwU7pZ8A+SLrA9M4TwM/X/PvXUTdXjW5mm1I3iDY7Nkl5ie0So/q66rLr5aQjh3wb47KKo/A4DnS9vc7yrUde1XKB4XnCy4j6Loausw1Y/Z8pNCOE+2xMlnSDpvx0fV663Pcmqh/J9ZpOkjkGcn7VLm98UNa7Pt8yiwV+nVzunqzuDWFTD/p1HtaSd6XjFR6tOtb1f6gnxQLrDV3p28TVpkT8o7nhSdpJmtGLvmqcdn/0+JE1fKOnIdBCQ7d2qLFtP+U9J2rVi2rmKdxB+mTKdRXOJ4veaqphIkuId4fc5vodinOKduVsrlhut2Etls6R/VLxjKlVfB9Vco3iQlCSlu6CVsj7jGkkfTSfs1errXkn7eeuvu2xzsK4WawjhGUnzJP1IBeslVsN1inc9xkiZ2269dXKv4jPdb0hl7Wq7Msk9WtJfU+JosuKdlwHjCCE8IWmd7TenSUVMpA7GLZLeZHt/KV482n5Z6h05OoQwV/F9H/226wHWwwrFd7eMsL23tvZ8e6HihdW6dPekZk+lGjGsULyTLkknKvZAkKTfS3p3+tyXKD7SIW1tuFenMtvlJdD17DPV3KRUX7Zfpvi47fJBLHOw4qNr0hC3n4KbJ+mfU68Fpe/zgjT9o976foc9U0+FLRzfQzEihPBrSf8h6fXpTyu0dZutd/srX+bdNeYrd5Okk1Obt4ekyVXm+V/F84Sxju++eb+kUg/Cb0q6SPExhHMGUf5ySeNsHypJtneyfVCdMVce17Pa7cr5strVqmzvq3jMP0fST7S1bh61/UrbIySdlFnAVo9KGu/4jqHnSXrbQAuUfa/SPnS8pBdnzLPNd0/t1/mKLwy/R/Ex0nrLv07S1NK2anu3tC6K6CZJJzm+Y21XSW9XfElv1vlyVTXquqTaucRsxWOVQgjLtveLDHP7lI4FiseX3+cZTCtzfKLi6RDChZK+rbgtr1B9bcN70vlQt+J7/mq187XOz8r3l3Zs8xsp6/q8JOu8bYH6X5sPtM/Uc31baPQ8ypBOLM6R9JEQwirbn1fsSjtFceP4ke0vK178XCLpj4ovWbzY9r8q9iCp5mpJ0x0fHViuuDMrhPCYY5fDy9KJ0F8lHVOxbD3lz5f0Rdt3Kt5p/YVit+nzVdBkRAhhWTrZeCiEUOou/xvFroB/VMzinhZCeMSxu2zJWZJ+nU5M5mvrXcO7JG20/UfFk4o7VN0pkn6Y6qJDcYeeXjFP1c8IIVydkk2LbG9QfF/Lv5d9p2dSfV5le7XiweTgKjFcIukcxxfbTQ0h9CpeALxLzXtsoqFS/f2XpBttb1Jc39MqZpstaZbtvyvWa1ZZGxxfHnem7edL+rvinYD1ZbNdJOlK24sUn7+/dxBxfERxP35a8YKyZaVjxjRJP0/HK0n6suLJwuWOvXaseJe/UtZ6+IPiOyOWKD5msTh91h9t36H4yFhfmq+WXTNiOCdNv1WxIS7ts79WvOu1VPH9cv8raV0I4Qnb56R4Vig+0tDyMrbVepyluB8tUbyDOS2E8Kxrd4X/kaTz03HuTqWTlO3cforqXMX3Bix2XCmPKb4T6Brbr5S0MK2r9ZI+qP6PAO2puJ5KN9VKvT2+LemXtv9RsUdoPb4q6Se2/11xW67HbxTv9i5R3AdurJwhhPCw7X9TbIssaW4I4XLbRyp2e39TCGGT7Xfb/kgI4fyByk/H3KmSvu/4SGuH4mPuA16EhxDW2P6D40uofyfpNFVvt9eof5uc1XZnOUrSDNvPKdbdh9L0Lyo+YvEXxWPHLgPE+5ztrynWyQNKbUcdvqq4nyxWXG9/rjJP1jnL6ZJuTo+s3Kn4CNpV9ZQfQrg7nWdek7bL5xR7xa2sM+4dJoSw2PGF/Hcqxndz+lPW+XKWo1S9rkuf02+bCyHMCPEVE/dI+m1Dv9TwdI+kDzu+uP1Piu3H2/MNqWW9SvEHcTYr7rv/rPjoaT1tw3LFY8FLJE1P1xRVZxzg/OxsSb+z/XAIYXIbtvmNVPX6vKTGNcYpiufTMxTPOT4ywOfUc31baKWXY6GNOf5Ky3dDCIfnHQsG5viLWKNDCP+RdyzAjmJ7lxDC+nRX51bFC+FH8o4LAFBcjj3Al0h6fel9MBi8dPE6J4RQ7UYndhDHx87mhBAuHWheIA/0PGpztr+omO1u9Ud0hgXbv5HUrWH4vDGGvTm2X6T4HqT/JHEEAKjF9tGKPwzzf0kcAUDz0fMIAAAAAAAAmXhhNgAAAAAAADKRPAIAAAAAAEAmkkcAAAAAAADIRPIIAACgQWzPTS9/rzXP+ozps21PbU5kAAAAQ8evrQEAAGwn21b8IZIT8o4FAACg0eh5BAAAkNj+pu1Plo2fYfsrtq+zvdj2Etsnpr+91PY9ts+StFjS3rZX2B6b/v5b27fbXmb7ExWf851U3nW2x1WJY6LtG9Py82zv0dxvDgAAkI3kEQAAwFaXSHpf2fh7JZ0v6aQQwuslTZb0ndTTSJJeLulnIYTXhRBWVpT10RDCREmTJJ1ie0ya/gJJi1N5N0r6SvlCtneSdKakqWn58yT9V8O+IQAAwCDx2BoAAEASQrjD9njbXZLGSXpc0sOSvmv7CEmbJe0p6SVpkZUhhFsyijvF9klpeG9JB0hak8r4RZp+oaTLKpZ7uaSDJV2bclQjUwwAAAC5IHkEAADQ36WSpkraXbEn0gcUE0kTQwjP2V4hqTPN+7dqBdg+StLRkg4NITxt+4ayZSqFysUlLQshHLod3wEAAKBheGwNAACgv0sknayYQLpU0mhJf02Jo8mS9q2jjNGSHk+Jo1dIOqTsbyNS2ZLUI+n3FcsulzTO9qFSfIzN9kFD/jYAAADbiZ5HAAAAZUIIy2zvKumhEMLDti+SdKXtRZLulHRvHcVcLWm67bsUk0Hlj7b9TdJBtm+XtE7937GkEMIG21Mlfd/2aMXzte9JWra93w0AAGAoHEJlT2kAAAAAAAAg4rE1AAAAAAAAZCJ5BAAAAAAAgEwkjwAAAAAAAJCJ5BEAAAAAAAAykTwCAAAAAABAJpJHAAAAAAAAyETyCAAAAAAAAJlIHgEAAAAAACDT/wfxVunPK3n7UQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#detecting and removing outliers\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig_size = (20, 10)\n",
    "fig, ax = plt.subplots(figsize = fig_size)\n",
    "sns.boxplot(x=\"variable\", y=\"value\", data=pd.melt(df2), ax=ax)\n",
    "\n",
    "#IQR Score\n",
    "quantile1 = data.quantile(0.25)\n",
    "quantile3 = data.quantile(0.75)\n",
    "IQR = quantile3 - quantile1\n",
    "\n",
    "print(IQR)\n",
    "\n",
    "#removing outliers\n",
    "data = data[~((data < (quantile1 - 1.5 * IQR)) |(data > (quantile3 + 1.5 * IQR))).any(axis=1)]\n",
    "\n",
    "print(data.shape)\n",
    "data.reset_index(inplace=True)\n",
    "data.drop('index', axis=1, inplace=True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classes:  [3 1 0 2]\n",
      "Class-0 Record Count:  98\n",
      "Class-1 Record Count:  147\n",
      "Class-2 Record Count:  169\n",
      "Class-3 Record Count:  156\n",
      "X:  [[ 0.         -0.02905208 -0.14262312 ...  1.83330482 -0.03772282\n",
      "  -0.37234207]\n",
      " [ 0.          1.46403379  2.87725703 ... -0.73156938  0.21724168\n",
      "  -0.76115528]\n",
      " [ 0.         -0.57199239  0.09896731 ... -0.73156938 -0.20769915\n",
      "  -1.22773113]\n",
      " ...\n",
      " [ 0.          0.24241808 -0.62580393 ... -0.93405947 -1.22755716\n",
      "   1.18291074]\n",
      " [ 0.          0.37815317 -0.38421352 ... -0.25909255 -0.20769915\n",
      "   0.17199641]\n",
      " [ 0.         -0.30052223  0.82373853 ...  0.68586111  0.04726535\n",
      "  -0.68339264]]\n",
      "R:  [[0 0 0 1]\n",
      " [0 1 0 0]\n",
      " [0 1 0 0]\n",
      " ...\n",
      " [0 0 0 1]\n",
      " [1 0 0 0]\n",
      " [0 1 0 0]]\n"
     ]
    }
   ],
   "source": [
    "print(\"Classes: \", data['quality_class'].unique())\n",
    "\n",
    "class_0_count = 0\n",
    "class_1_count = 0\n",
    "class_2_count = 0\n",
    "class_3_count = 0\n",
    "\n",
    "R = []\n",
    "X = []\n",
    "\n",
    "for i in range(0, len(data)):\n",
    "    if(data['quality_class'][i] == 0): \n",
    "        R.append([1, 0, 0, 0])\n",
    "        class_0_count = class_0_count + 1\n",
    "    elif(data['quality_class'][i] == 1): \n",
    "        R.append([0, 1, 0, 0])    \n",
    "        class_1_count = class_1_count + 1\n",
    "    elif(data['quality_class'][i] == 2): \n",
    "        R.append([0, 0, 1, 0])\n",
    "        class_2_count = class_2_count + 1\n",
    "    elif(data['quality_class'][i] == 3): \n",
    "        R.append([0, 0, 0, 1])        \n",
    "        class_3_count = class_3_count + 1\n",
    "    \n",
    "    X.append([data['fixed acidity'][i], data['volatile acidity'][i], data['citric acid'][i], data['residual sugar'][i], data['chlorides'][i], data['free sulfur dioxide'][i], data['total sulfur dioxide'][i], data['density'][i], data['pH'][i], data['sulphates'][i], data['alcohol'][i]])    \n",
    "\n",
    "print(\"Class-0 Record Count: \", class_0_count)\n",
    "print(\"Class-1 Record Count: \", class_1_count)\n",
    "print(\"Class-2 Record Count: \", class_2_count)\n",
    "print(\"Class-3 Record Count: \", class_3_count)\n",
    "\n",
    "N = len(X)\n",
    "X = np.c_[np.ones(N), X]             #Adding Ones to the X as first bias input\n",
    "X = np.array(X)\n",
    "R = np.array(R)\n",
    "\n",
    "#data normalization\n",
    "#for i in range(0, len(X)):              #row\n",
    "#    for j in range(1, len(X[0]) - 1):   #column\n",
    "#        X[i, j] = round((X[i, j] - min(X[:, j])) / (max(X[:, j]) - min(X[:, j])), 9)    # x' = (x - min(x)) / (max(x) - min(x))       \n",
    "\n",
    "#data standardization\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X = scaler.transform(X)\n",
    "\n",
    "print(\"X: \", X)\n",
    "print(\"R: \", R)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP Backpropagation algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tanh(xx):\n",
    "    return 2 / (1 + math.exp(-2 * xx))\n",
    "    \n",
    "def mlp_back_prop(x, r, H, K, D, n, epochs, desired_accuracy, alpha, beta):\n",
    "    np.random.seed(100)        \n",
    "    w = np.random.uniform(-0.01, 0.01, (D + 1, H))                                                                #initialize all vih and whj to rand (-0.01, 0.01)\n",
    "    v = np.random.uniform(-0.01, 0.01, (H + 1, K))\n",
    "    error = []\n",
    "    prev_errors = []    \n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        output = []\n",
    "        accuracy = 0\n",
    "                \n",
    "        d_w = np.zeros((D + 1, H))\n",
    "        d_v = np.zeros((H + 1, K))\n",
    "                        \n",
    "        np.random.shuffle(x)\n",
    "        \n",
    "        for t in range(len(x)):                                                                                   #For all (xt, rt) ∈ X in random order            \n",
    "            w_x = np.dot(w.T, x[t, :])\n",
    "            zh = np.append(1, list(map(np.tanh, w_x)))                                                                 #For h = 1,....,H   zh = tanh(whT . xt)                                \n",
    "            \n",
    "            y_temp = np.dot(v.T, zh)                                                                                   #For i = 1,....,K   yi = viT . z            \n",
    "            y_temp2 = [math.exp(i) for i in y_temp]\n",
    "            y = [(i / sum(y_temp2)) for i in y_temp2]\n",
    "            \n",
    "            if(np.argmax(y) == np.argmax(r[t])):\n",
    "                accuracy = accuracy + 1\n",
    "                \n",
    "            for i in range(K):                                                                                         #For i = 1,....,K\n",
    "                d_v[:, i] = n * np.dot((r[t, i] - y[i]), zh)                                                                #delta_vi = eta(rit - yit) . z\n",
    "            \n",
    "            for h in range(H):                                                                                         #For h = 1,....,H\n",
    "                d_w[:, h] = n * np.dot((r[t, ] - y), v[h + 1, :]) * zh[h + 1] * (1 - zh[h + 1]) * X[t, :]         #delta_wh = eta(SUMi(rit - yit) * vih) * zh * (1 - zh) * xt                                \n",
    "                \n",
    "            v = v + d_v                                                                                                #For i = 1,....,K   vi = vi + delta_vi\n",
    "            w = w + d_w                                                                                                #For h = 1,....,H   wh = wh + delta_wh\n",
    "            \n",
    "            output.append(y)\n",
    "        \n",
    "        error.append(1 - (accuracy / len(x)))\n",
    "                \n",
    "        ############ Adaptive learning ###############\n",
    "        \n",
    "        if(epoch >= 1):\n",
    "            if(error[epoch] < np.average(prev_errors)):      #comparing with average of last 3 errors\n",
    "                n = n + alpha\n",
    "            else:\n",
    "                n = n - (beta * n)\n",
    "                \n",
    "            if len(prev_errors) > 3:\n",
    "                prev_errors.pop(0)\n",
    "               \n",
    "        prev_errors.append(error[epoch])\n",
    "        ##############################################\n",
    "        \n",
    "        if(accuracy / len(x) >= desired_accuracy):\n",
    "            print(\"Achieved Accuracy: \", round(accuracy / len(x) * 100, 2))\n",
    "            print(\"Desired Accuracy: \", round(desired_accuracy * 100, 2))\n",
    "            return (v, w)\n",
    "        else: \n",
    "            print(\"EPOCH: {}\".format(epoch + 1), \"   Accuracy: {}\".format(round(accuracy / len(x) * 100, 2)))\n",
    "\n",
    "    print(\"Achieved Accuracy: \", round(accuracy / len(x) * 100, 2))\n",
    "    print(\"Desired Accuracy: \", round(desired_accuracy * 100, 2))\n",
    "        \n",
    "    return(v, w, error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different configurations No. 1 obtained using number of hidden units = 4000 and initial learning rate = 0.0005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1    Accuracy: 29.47\n",
      "EPOCH: 2    Accuracy: 30.0\n",
      "EPOCH: 3    Accuracy: 29.12\n",
      "EPOCH: 4    Accuracy: 29.82\n",
      "EPOCH: 5    Accuracy: 29.65\n",
      "EPOCH: 6    Accuracy: 29.47\n",
      "EPOCH: 7    Accuracy: 28.77\n",
      "EPOCH: 8    Accuracy: 30.18\n",
      "EPOCH: 9    Accuracy: 29.12\n",
      "EPOCH: 10    Accuracy: 29.65\n",
      "EPOCH: 11    Accuracy: 29.65\n",
      "EPOCH: 12    Accuracy: 29.65\n",
      "EPOCH: 13    Accuracy: 29.65\n",
      "EPOCH: 14    Accuracy: 29.65\n",
      "EPOCH: 15    Accuracy: 29.65\n",
      "EPOCH: 16    Accuracy: 29.65\n",
      "EPOCH: 17    Accuracy: 29.65\n",
      "EPOCH: 18    Accuracy: 29.65\n",
      "EPOCH: 19    Accuracy: 29.65\n",
      "EPOCH: 20    Accuracy: 29.65\n",
      "EPOCH: 21    Accuracy: 29.65\n",
      "EPOCH: 22    Accuracy: 29.65\n",
      "EPOCH: 23    Accuracy: 29.65\n",
      "EPOCH: 24    Accuracy: 29.65\n",
      "EPOCH: 25    Accuracy: 29.65\n",
      "EPOCH: 26    Accuracy: 29.65\n",
      "EPOCH: 27    Accuracy: 29.65\n",
      "EPOCH: 28    Accuracy: 29.65\n",
      "EPOCH: 29    Accuracy: 29.65\n",
      "EPOCH: 30    Accuracy: 29.65\n",
      "EPOCH: 31    Accuracy: 29.65\n",
      "EPOCH: 32    Accuracy: 29.65\n",
      "EPOCH: 33    Accuracy: 29.65\n",
      "EPOCH: 34    Accuracy: 29.65\n",
      "EPOCH: 35    Accuracy: 29.65\n",
      "EPOCH: 36    Accuracy: 29.65\n",
      "EPOCH: 37    Accuracy: 29.65\n",
      "EPOCH: 38    Accuracy: 29.65\n",
      "EPOCH: 39    Accuracy: 29.65\n",
      "EPOCH: 40    Accuracy: 29.65\n",
      "Achieved Accuracy:  29.65\n",
      "Desired Accuracy:  95.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([[-2.70058495e-01,  3.74971601e-02,  1.51368139e-01,\n",
       "          8.99354542e-02],\n",
       "        [ 7.94290760e-03,  4.58337022e-03, -9.10304576e-03,\n",
       "          5.84260967e-03],\n",
       "        [ 6.88481362e-03,  9.61312356e-03,  5.52628276e-03,\n",
       "         -2.52846837e-03],\n",
       "        ...,\n",
       "        [ 5.19636825e-03,  2.42318730e-03, -3.09661794e-05,\n",
       "         -3.19534966e-03],\n",
       "        [-5.79412508e-03, -6.57609094e-03, -5.68454289e-03,\n",
       "          8.88890468e-03],\n",
       "        [-2.28908205e-04,  4.49727834e-03, -3.44731648e-03,\n",
       "         -6.39497383e-03]]),\n",
       " array([[ 8.68098836e-04, -4.43261230e-03, -1.50964819e-03, ...,\n",
       "          5.86577655e-03, -3.03361889e-03,  9.32333944e-03],\n",
       "        [ 5.34223531e-03, -9.12331171e-03, -6.17343244e-03, ...,\n",
       "          5.22064507e-03,  9.68449957e-04, -1.19029419e-05],\n",
       "        [-7.87709261e-03, -9.91078620e-03,  2.54990074e-03, ...,\n",
       "         -8.67186677e-03, -5.65045564e-03,  7.94791688e-03],\n",
       "        ...,\n",
       "        [-6.67490793e-03, -5.86784472e-03, -1.29385397e-03, ...,\n",
       "          2.25759009e-03, -4.74370896e-03,  1.68024710e-03],\n",
       "        [-5.88969757e-03,  2.71238333e-03, -2.20407034e-03, ...,\n",
       "          6.05049904e-03,  4.86260173e-03,  3.60255851e-03],\n",
       "        [-3.79161803e-03, -5.34168680e-03, -6.72026523e-03, ...,\n",
       "          8.44649325e-03, -9.91198642e-03, -9.30354629e-03]]),\n",
       " [0.7052631578947368,\n",
       "  0.7,\n",
       "  0.7087719298245614,\n",
       "  0.7017543859649122,\n",
       "  0.7035087719298245,\n",
       "  0.7052631578947368,\n",
       "  0.712280701754386,\n",
       "  0.6982456140350877,\n",
       "  0.7087719298245614,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245,\n",
       "  0.7035087719298245])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Hyper-parameters\n",
    "N = len(X) #number of samples\n",
    "K = len(data['quality_class'].unique())  #number of classes\n",
    "D = len(X[0, :]) - 1  #dimension or number of independent variables\n",
    "H = 4000 #size of hidden layer\n",
    "eta = 0.0005 #eta\n",
    "iterations = 40\n",
    "accuracy_threshold = 0.95\n",
    "alpha = 0.001\n",
    "beta = 0.9\n",
    "\n",
    "#returns/prints accuracy per epoch, accuracy achieved, desired accuracy, V matrix, W matrix, Error matrix\n",
    "mlp_back_prop(X, R, H, K, D, eta, iterations, accuracy_threshold, alpha, beta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different configurations No. 2 obtained using number of hidden units = 3500 and initial learning rate = 0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EPOCH: 1    Accuracy: 24.91\n",
      "EPOCH: 2    Accuracy: 26.32\n",
      "EPOCH: 3    Accuracy: 29.3\n",
      "EPOCH: 4    Accuracy: 29.47\n",
      "EPOCH: 5    Accuracy: 29.3\n",
      "EPOCH: 6    Accuracy: 30.53\n",
      "EPOCH: 7    Accuracy: 29.12\n",
      "EPOCH: 8    Accuracy: 29.65\n",
      "EPOCH: 9    Accuracy: 27.02\n",
      "EPOCH: 10    Accuracy: 27.37\n",
      "EPOCH: 11    Accuracy: 31.75\n",
      "EPOCH: 12    Accuracy: 28.42\n",
      "EPOCH: 13    Accuracy: 28.42\n",
      "EPOCH: 14    Accuracy: 31.23\n",
      "EPOCH: 15    Accuracy: 29.12\n",
      "EPOCH: 16    Accuracy: 30.53\n",
      "EPOCH: 17    Accuracy: 28.25\n",
      "EPOCH: 18    Accuracy: 29.82\n",
      "EPOCH: 19    Accuracy: 26.67\n",
      "EPOCH: 20    Accuracy: 31.23\n",
      "EPOCH: 21    Accuracy: 28.25\n",
      "EPOCH: 22    Accuracy: 29.82\n",
      "EPOCH: 23    Accuracy: 28.6\n",
      "EPOCH: 24    Accuracy: 29.47\n",
      "EPOCH: 25    Accuracy: 28.07\n",
      "EPOCH: 26    Accuracy: 30.35\n",
      "EPOCH: 27    Accuracy: 27.37\n",
      "EPOCH: 28    Accuracy: 30.0\n",
      "EPOCH: 29    Accuracy: 29.82\n",
      "EPOCH: 30    Accuracy: 29.3\n",
      "EPOCH: 31    Accuracy: 27.37\n",
      "EPOCH: 32    Accuracy: 28.6\n",
      "EPOCH: 33    Accuracy: 29.82\n",
      "EPOCH: 34    Accuracy: 28.25\n",
      "EPOCH: 35    Accuracy: 29.47\n",
      "EPOCH: 36    Accuracy: 30.7\n",
      "EPOCH: 37    Accuracy: 31.23\n",
      "EPOCH: 38    Accuracy: 29.12\n"
     ]
    }
   ],
   "source": [
    "#Hyper-parameters\n",
    "H = 3500 #size of hidden layer\n",
    "eta = 0.0001 #eta\n",
    "\n",
    "#returns/prints accuracy per epoch, accuracy achieved, desired accuracy, V matrix, W matrix, Error matrix\n",
    "mlp_back_prop(X, R, H, K, D, eta, iterations, accuracy_threshold, alpha, beta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different configurations No. 3 obtained using number of hidden units = 3000 and initial learning rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyper-parameters\n",
    "H = 3000 #size of hidden layer\n",
    "eta = 0.001 #eta\n",
    "\n",
    "#returns/prints accuracy per epoch, accuracy achieved, desired accuracy, V matrix, W matrix, Error matrix\n",
    "mlp_back_prop(X, R, H, K, D, eta, iterations, accuracy_threshold, alpha, beta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different configurations No. 3 obtained using number of hidden units = 2000 and initial learning rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyper-parameters\n",
    "H = 2000 #size of hidden layer\n",
    "eta = 0.01 #eta\n",
    "\n",
    "#returns/prints accuracy per epoch, accuracy achieved, desired accuracy, V matrix, W matrix, Error matrix\n",
    "mlp_back_prop(X, R, H, K, D, eta, iterations, accuracy_threshold, alpha, beta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different configurations No. 5 obtained using number of hidden units = 1000 and initial learning rate = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Hyper-parameters\n",
    "H = 1000 #size of hidden layer\n",
    "eta = 0.1 #eta\n",
    "\n",
    "#returns/prints accuracy per epoch, accuracy achieved, desired accuracy, V matrix, W matrix, Error matrix\n",
    "mlp_back_prop(X, R, H, K, D, eta, iterations, accuracy_threshold, alpha, beta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "print(\"Classes: \", data['quality_class'].unique())\n",
    "\n",
    "class_0_count = 0\n",
    "class_1_count = 0\n",
    "class_2_count = 0\n",
    "class_3_count = 0\n",
    "\n",
    "R = []\n",
    "X = []\n",
    "\n",
    "for i in range(0, len(data)):\n",
    "    if(data['quality_class'][i] == 0): \n",
    "        R.append([1, 0, 0, 0])\n",
    "        class_0_count = class_0_count + 1\n",
    "    elif(data['quality_class'][i] == 1): \n",
    "        R.append([0, 1, 0, 0])    \n",
    "        class_1_count = class_1_count + 1\n",
    "    elif(data['quality_class'][i] == 2): \n",
    "        R.append([0, 0, 1, 0])\n",
    "        class_2_count = class_2_count + 1\n",
    "    elif(data['quality_class'][i] == 3): \n",
    "        R.append([0, 0, 0, 1])        \n",
    "        class_3_count = class_3_count + 1\n",
    "    \n",
    "    X.append([data['fixed acidity'][i], data['volatile acidity'][i], data['citric acid'][i], data['residual sugar'][i], data['chlorides'][i], data['free sulfur dioxide'][i], data['total sulfur dioxide'][i], data['density'][i], data['pH'][i], data['sulphates'][i], data['alcohol'][i]])    \n",
    "\n",
    "print(\"Class-0 Record Count: \", class_0_count)\n",
    "print(\"Class-1 Record Count: \", class_1_count)\n",
    "print(\"Class-2 Record Count: \", class_2_count)\n",
    "print(\"Class-3 Record Count: \", class_3_count)\n",
    "\n",
    "N = len(X)\n",
    "X = np.c_[np.ones(N), X]             #Adding Ones to the X as first bias input\n",
    "X = np.array(X)\n",
    "R = np.array(R)            \n",
    "\n",
    "#data standardization\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X)\n",
    "X = scaler.transform(X)\n",
    "\n",
    "X_test = X[0:400, :]\n",
    "Y_test = R[0:400, :]\n",
    "\n",
    "print(X_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(hidden_layer_sizes=(400), solver='adam', learning_rate_init=0.0005, learning_rate='adaptive', activation='relu', max_iter=1000, random_state=42)\n",
    "scores = cross_validate(mlp.fit(X, R), np.array(X), np.array(R))\n",
    "\n",
    "print(scores)\n",
    "\n",
    "pred = mlp.predict(X_test)\n",
    "\n",
    "print('Accuracy: {}'.format(round(accuracy_score(np.array(Y_test), np.array(pred)) * 100, 2)))\n",
    "\n",
    "classes = ['Class-0', 'Class-1', 'Class-2', 'Class-3']\n",
    "print(classification_report(np.array(Y_test), np.array(pred), target_names=classes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different config to get better performance: performance increased by increasing number neurons in hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(hidden_layer_sizes=(500), solver='adam', learning_rate_init=0.0005, learning_rate='adaptive', activation='relu', max_iter=1000, random_state=42)\n",
    "scores = cross_validate(mlp.fit(X, R), np.array(X), np.array(R))\n",
    "\n",
    "print(scores)\n",
    "\n",
    "pred = mlp.predict(X_test)\n",
    "\n",
    "print('Accuracy: {}'.format(round(accuracy_score(np.array(Y_test), np.array(pred)) * 100, 2)))\n",
    "\n",
    "classes = ['Class-0', 'Class-1', 'Class-2', 'Class-3']\n",
    "print(classification_report(np.array(Y_test), np.array(pred), target_names=classes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 80/20 Train Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "print(\"Classes: \", data['quality_class'].unique())\n",
    "\n",
    "R = []\n",
    "X = []\n",
    "\n",
    "for i in range(0, len(data)):\n",
    "    if(data['quality_class'][i] == 0): R.append([1, 0, 0, 0])\n",
    "    elif(data['quality_class'][i] == 1): R.append([0, 1, 0, 0])    \n",
    "    elif(data['quality_class'][i] == 2): R.append([0, 0, 1, 0])\n",
    "    elif(data['quality_class'][i] == 3): R.append([0, 0, 0, 1])        \n",
    "    \n",
    "    X.append([data['fixed acidity'][i], data['volatile acidity'][i], data['citric acid'][i], data['residual sugar'][i], data['chlorides'][i], data['free sulfur dioxide'][i], data['total sulfur dioxide'][i], data['density'][i], data['pH'][i], data['sulphates'][i], data['alcohol'][i]])    \n",
    "\n",
    "N = len(X)\n",
    "X = np.c_[np.ones(N), X]             #Adding Ones to the X as first bias input\n",
    "X = np.array(X)\n",
    "R = np.array(R)\n",
    "       \n",
    "print(\"X: \", X)\n",
    "print(\"R: \", R)\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, R, test_size=0.2, random_state=0)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "\n",
    "print('Train Dataset: ', X_train.shape)\n",
    "print('Test Dataset: ', X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(hidden_layer_sizes=(600), solver='adam', learning_rate_init=0.0005, learning_rate='adaptive', activation='relu', max_iter=1000, random_state=42)\n",
    "mlp.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = mlp.predict(X_test)\n",
    "\n",
    "print('Accuracy: {}'.format(round(accuracy_score(np.array(Y_test), np.array(pred)) * 100, 2)))\n",
    "\n",
    "classes = ['Class-0', 'Class-1', 'Class-2', 'Class-3']\n",
    "print(classification_report(np.array(Y_test), np.array(pred), target_names=classes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Different config to get better performance: performance increased by increasing number of neurons in the hidden layer and initial learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(hidden_layer_sizes=(800), solver='adam', learning_rate_init=0.001, learning_rate='adaptive', activation='relu', max_iter=1000, random_state=42)\n",
    "mlp.fit(X_train, Y_train)\n",
    "\n",
    "pred = mlp.predict(X_test)\n",
    "\n",
    "print('Accuracy: {}'.format(round(accuracy_score(np.array(Y_test), np.array(pred)) * 100, 2)))\n",
    "\n",
    "classes = ['Class-0', 'Class-1', 'Class-2', 'Class-3']\n",
    "print(classification_report(np.array(Y_test), np.array(pred), target_names=classes))"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}