NN manual 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4lfWd9/H39+RkJyFC2AkENIBgRTQsLm1VtC51azt1qba1OjLtVEfbXp3HTmfs1HlmptNO16d2cZTauoFbLTOlrmjrBgQEld3IlrAGAgTInvN9/jiHTIzs5s59ls/runKdc+5zJ/lccHI+53f/7sXcHREREYBI2AFERCR5qBRERKSTSkFERDqpFEREpJNKQUREOqkURESkk0pBREQ6qRRERKSTSkFERDpFww5wrEpLS728vDzsGCIiKWXx4sU73H3AkdZLuVIoLy9n0aJFYccQEUkpZrbhaNbT5iMREemkUhARkU4qBRER6aRSEBGRTioFERHpFFgpmNlMM9tuZssO8byZ2c/MrNrM3jaz04PKIiIiRyfIkcIDwMWHef4SoCLxNQP4ZYBZRETkKAR2nIK7/8XMyg+zypXA7zx+PdD5ZlZiZkPcfUtQmUREDsfdaetwWjtitLR10NoRo7U98dURoyPmH/xypz3mxGLx2yOt4+7EHDzx+9wh5o6TuPWuy8GJr487008exMSykkD/DcI8eG0YUNPlcW1i2QdKwcxmEB9NMGLEiF4JJyKpoam1g12NrdTvb2V3Yxv1ja00NLWxv6Wd/a0d7G9pp7G1nX0t8fvx5e00tnTQ0h6jpT1+e+CNP1kvW28Gg/rmpXUpHDV3vxe4F6CysjJJ/8tEpCc1t3VQU9/I5j3NbN3TxNY9LWxtaGLrnma2NbSwq7GVXY2tNLfFDvtzCnOyKMiN0ic3SmFuFgU5UQYW5ZHfP4u8aBa52RFysiLkRuNfOYmv3GhW/H5WhNzsCNFIhGjEyMoyssyIRoxIpNutGdGsLvcjESIROm+zLL7cDCxxGzHDoMtyMIxIYp0Dt70lzFLYBJR1eTw8sUxEMsj2hmaWb26gevs+1u3cz/od8a8tDc0f+NRe2ieHwX3zGNI3j/FDi+lXmENJQTb9CnIoKcihX2EOJxRk0zc/m8LcKPnZWUQivfeGmg7CLIU5wK1mNguYCuzRfIJIetu8u4klG3ezfPMelm9uYPnmBnbsa+l8vqQgm/L+hUwd3Z/y/oWUlxYwrCSfQcV5DCrOIyeqveiDFlgpmNmjwLlAqZnVAt8BsgHc/VfAXOBSoBpoBL4UVBYRCceWPU28smYHC9bVs2DdTmp3NQEQjRgVg4o4d+wAJgwtZsLQvowZ1IeSgpyQE0uQex9dd4TnHfhqUL9fRHpfLOYsrd3NvJXbeXHVdlZuaQCgX2EOU8r7cdPZo6gsP4Gxg4vIjWaFnFYOJiUmmkUkua3eupenl25iztLNbNrdRFbEOGPkCdx5yTjOHTuAsYOKenWyVI6fSkFEjktTawdPL93Eg29sYMWWBrIixjknlfKNT4xh+rhB9C3IDjuiHAeVgogck5r6Rh6cv4HZVTXsaWpj3OAi/vny8Xzy1KEMKMoNO558SCoFETkqm3Y38fN57/L4olocuHjCYL54VjmTy0/QpqE0olIQkcPa3tDMz1+qZtbC+AkIPjd1BF/++IkMLckPOZkEQaUgIgfV3hHjt29s4MfPr6G5rYPPVg7n1vMrGKYySGsqBRH5gMUb6vn275exautePj5mAN+9YgLlpYVhx5JeoFIQkU7NbR1870+reOD19Qzpm8evbjidiyYM1pxBBlEpiAgA727by22PLmHV1r3ceFY537xoLIW5eovINPofF8lw7s6sqhq++9/LKcyJ8psvTea8sQPDjiUhUSmIZLDW9hjf/v07PL64lo9WlPLDqycysCgv7FgSIpWCSIba09jGlx9azBtrd/J30yu4Y3qFTjMtKgWRTLRxZyNfemAhG+sb+dHVE/n06cPDjiRJQqUgkmGWbdrDF2YuJObOQzdPZero/mFHkiSiUhDJIMs37+GG+xdQmBPlwZunMHpAn7AjSZJRKYhkiBWbG7j+vgUUZGfx6C3TGNG/IOxIkoR0bTuRDBAvhPnkZ2fx6AwVghyaSkEkza3fsZ8b7l9AXnYWs2ZMY2R/na5CDk2lIJLGdje2ctMDVbg7j9yiQpAj05yCSJpqbY/x5YcWU7uriYf+eiqjdEI7OQoqBZE05O5866l3mL+2nh9fM5Epo/qFHUlShDYfiaShX7z8Hk++Wcvt0yv41CQdmCZHT6UgkmZeq97Bfz63mismDuWOCyrCjiMpRqUgkkbq9rZwx+yljC4t5Huf+YiugyDHTHMKImkiFnO+/thSGpra+N1NUyjI0Z+3HDuNFETSxK/+8h6vvLuD71w+gZOHFIcdR1KUSkEkDSzeUM8Pn1vDJ08dwnVTysKOIylMpSCS4ppaO/ja7LcYVpLPv39a8wjy4Wijo0iK++Fzq9lY38isGdMozssOO46kOI0URFLYko27mPnaOq6fOoJpui6C9IBAS8HMLjaz1WZWbWZ3HuT5EWb2kpktMbO3zezSIPOIpJPW9hj/58m3GVScx52XjAs7jqSJwErBzLKAe4BLgPHAdWY2vttq/wg85u6TgGuBXwSVRyTd/OLlatZs28e/fuoUirTZSHpIkCOFKUC1u69191ZgFnBlt3UcOLDvXF9gc4B5RNLG6q17ueelaq46bSjnjxsUdhxJI0FONA8Daro8rgWmdlvnn4HnzOw2oBC4IMA8ImnB3fmnPyyjT26Uuy6fEHYcSTNhTzRfBzzg7sOBS4EHzewDmcxshpktMrNFdXV1vR5SJJn8adlWFq6r5xufGEu/wpyw40iaCbIUNgFdj6IZnljW1c3AYwDu/gaQB5R2/0Hufq+7V7p75YABAwKKK5L8mts6+Le5Kxk3uIhrJ+sgNel5QZZCFVBhZqPMLIf4RPKcbutsBKYDmNnJxEtBQwGRQ7j/1XXU7mrirsvGE80Ke6Av6SiwV5W7twO3As8CK4nvZbTczO42sysSq30DuMXM3gIeBW50dw8qk0gq29bQzD0vVXPRhEGcddIHBtQiPSLQI5rdfS4wt9uyu7rcXwGcHWQGkXTx/WdW097hfPvS7nt2i/QcjT9FUsA7tXt48s1abjpnFCP6F4QdR9KYSkEkBXz/2VX0K8zhq+edGHYUSXMqBZEkt2DtTl55dwdf+fiJOnJZAqdSEEli7s4Pn1vDwKJcbpg2Muw4kgFUCiJJ7NXqHSxcX8+t559Efk5W2HEkA6gURJKUu/Ofz65mWEk+1+hANeklKgWRJPXCyu28VbuH26dXkBvVKEF6h0pBJAnFYs4Pn1vNqNJCPn36sLDjSAZRKYgkoedWbGPV1r3ccUGFTmchvUqvNpEk4+788uVqRvYv4JMfGRJ2HMkwKgWRJPP6ezt5q3YPf/OxEzVKkF6nV5xIkvnly+8xoChXcwkSCpWCSBJ5u3Y3r1bv4K/PGUVetvY4kt6nUhBJIr946T2K86J8buqIsKNIhlIpiCSJ6u37eHbFVr5wZrnOcSShUSmIJIlf//k9crIi3Hh2edhRJIOpFESSwLaGZp5euolrJpdR2ic37DiSwVQKIkngofkbaI85N58zKuwokuFUCiIha27r4OEFG5k+bhAj+xeGHUcynEpBJGR/WLqJ+v2t3HROedhRRFQKImFyd2a+up5xg4s4c3T/sOOIqBREwvTGeztZvW0vN50zCjMLO46ISkEkTDNfW0f/whyumDg07CgigEpBJDTrduznxVXbuX7qCJ3SQpKGSkEkJL99fT3RiHHDtJFhRxHppFIQCcG+lnaeWFzLZacOZWBxXthxRDqpFERC8PSSTexraefzZ2qUIMlFpSDSy9ydh+ZvYPyQYiaVlYQdR+R9VAoivWzxhl2s2rqXz585UruhStJRKYj0sofmb6AoN8qVp2k3VEk+KgWRXrRzXwtz39nKZ84YTkFONOw4Ih8QaCmY2cVmttrMqs3szkOsc7WZrTCz5Wb2SJB5RML22KJaWjtiXK8rq0mSCuyjipllAfcAFwK1QJWZzXH3FV3WqQC+BZzt7rvMbGBQeUTC1hFzHlm4gWmj+1ExqCjsOCIHFeRIYQpQ7e5r3b0VmAVc2W2dW4B73H0XgLtvDzCPSKj+sqaOmvomHawmSS3IUhgG1HR5XJtY1tUYYIyZvWZm883s4oP9IDObYWaLzGxRXV1dQHFFgvXwgg2U9snlE+MHhx1F5JDCnmiOAhXAucB1wH+Z2Qd23Hb3e9290t0rBwwY0MsRRT68rXuambdqO1dXDicnGvafncihBfnq3ASUdXk8PLGsq1pgjru3ufs6YA3xkhBJK0++WUvM4erKsiOvLBKiIEuhCqgws1FmlgNcC8zpts7TxEcJmFkp8c1JawPMJNLrYjFndlUNZ47uT3mpLrcpyS2wUnD3duBW4FlgJfCYuy83s7vN7IrEas8CO81sBfAS8E133xlUJpEwzF+7k431jVw7RaMESX6BHj3j7nOBud2W3dXlvgNfT3yJpKVZVTX0zc/mogmaYJbkpxkvkQDt2t/KM8u28qlJw3QhHUkJKgWRAD29dBOtHTGumaxNR5IaVAoiAXF3Zi2sYWJZCScPKQ47jshRUSmIBGRpzW5Wb9vLtRolSApRKYgEZHZVDQU5WVw+UafIltShUhAJwL6Wdua8tZnLTh1Cn1ydIltSh0pBJAB/fHszja0dXDNZp8iW1KJSEAnArKoaKgb24fQRugazpJajGtcmrnNwNjAUaAKWAYvcPRZgNpGUtHrrXpZs3M0/fvJkXYNZUs5hS8HMzgPuBPoBS4DtQB5wFXCimT0B/NDdG4IOKpIqZlfVkJ1lfPr04WFHETlmRxopXArc4u4buz9hZlHgMuJXVnsygGwiKaelvYOnltTyiQmD6VeYE3YckWN22FJw928e5rl24mc5FZGE55ZvY3djm45NkJR1VBPNZvagmfXt8rjczF4MLpZIappdVcOwknzOPrE07Cgix+Vo9z56FVhgZpea2S3Ac8BPgoslkno27mzk1eodXDO5jEhEE8ySmo5q7yN3/7WZLSd+zYMdwCR33xpoMpEU89iiGiIGn63UBLOkrqPdfPR5YCbwBeABYK6ZTQwwl0hKae+I8fjiGs4dO5AhffPDjiNy3I72+PvPAOe4+3bgUTP7PfFymBRUMJFU8uc1dWxraOHuKzXBLKntaDcfXdXt8UIzmxpMJJHUM6uqhtI+uZw/bmDYUUQ+lMNuPjKzfzSzfgd7zt1bzex8M7ssmGgiqWF7QzPzVm3nr84YTnaWzhwjqe1II4V3gP82s2bgTaCO+BHNFcBpwAvAvwWaUCTJPfFmLR0x19XVJC0cqRT+yt3PNrO/J36KiyFAA/AQMMPdm4IOKJLM3J3ZVTVMHdWPUaWFYccR+dCOVApnmNlQ4HrgvG7P5RM/OZ5Ixpq/tp4NOxu544KKsKOI9IgjlcKvgBeB0cCiLssN8MRykYw1u2ojRXlRLjllSNhRRHrEYWfF3P1n7n4yMNPdR3f5GuXuKgTJaLsbW5m7bCufmjSMvOyssOOI9Iij2lXC3b8SdBCRVPPUm5tobY9pglnSivafEzkO7s6jCzcysayECUP7HvkbRFKESkHkOCzesIt3t+/j+im6BrOkF5WCyHF4ZMFG+uRGuWyiJpglvagURI7R7sZW/uedLVw1aSgFOUd7+jCR1KBSEDlGv18Sn2D+3JSRYUcR6XGBloKZXWxmq82s2szuPMx6nzEzN7PKIPOIfFjuziML4hPM44cWhx1HpMcFVgpmlgXcA1wCjAeuM7PxB1mvCLgdWBBUFpGecmCC+XNTtBuqpKcgRwpTgGp3X+vurcAs4MqDrPcvwH8AzQFmEekRjyyMTzBfPnFo2FFEAhFkKQwDaro8rk0s62RmpwNl7v7HAHOI9Ig9jW388W1NMEt6C22i2cwiwI+AbxzFujPMbJGZLaqrqws+nMhBPLWklpb2GNfp2ARJY0GWwiag64bX4YllBxQBpwAvm9l6YBow52CTze5+r7tXunvlgAEDAowscnA6glkyRZClUAVUmNkoM8sBrgXmHHjS3fe4e6m7l7t7OTAfuMLdFx38x4mEZ/GGXazZpglmSX+BlYK7twO3As8CK4HH3H25md1tZlcE9XtFgnBggvmyUzXBLOkt0Nkyd58LzO227K5DrHtukFlEjteu/a388e0tfLZyOIW5mmCW9KYjmkWOYFZVDS3tMT4/rTzsKCKBUymIHEZHzHlo/gbOHN2fsYOLwo4jEjiVgshhvLhyG5t2N/HFs3SeI8kMKgWRw/jtG+sZ2jePC04eFHYUkV6hUhA5hOrte3mteifXTxtJNEt/KpIZ9EoXOYTfvr6BnGiEa3UNZskgKgWRg2hobuPJN2u5/NSh9O+TG3YckV6jUhA5iKcW19LY2qEJZsk4KgWRbjpizgOvr+e0shJOHV4SdhyRXqVSEOnm+RXbWL+zkVs+OjrsKCK9TqUg0s1/vbKWsn75XDRBu6FK5lEpiHSxeMMuFm/Yxc1nj9JuqJKR9KoX6eK+V9bSNz+bz1ZqN1TJTCoFkYQNO/fzzPKtXD91hM6GKhlLpSCScP+r64hGjBvPKg87ikhoVAoixK+Z8NiiGq46bRgDi/PCjiMSGpWCCPDg/A00t8W45WPaDVUym0pBMt6+lnbuf3Ud08cNZMwgXTNBMptKQTLe795Yz56mNv5uekXYUURCp1KQjLa/pZ37XlnHuWMHMLFMp7QQUSlIRnt4wQbq97dy2/kaJYiASkEyWFNrB/f+ZS0frSjljJEnhB1HJCmoFCRjPbJwIzv2tWouQaQLlYJkpOa2Dn715/c4c3R/Jpf3CzuOSNJQKUhGenjBRur2tnDb9JPCjiKSVFQKknEamtv4+bx3OeekUs46sTTsOCJJRaUgGedXL7/HrsY27rxkXNhRRJKOSkEyypY9Tdz/6jquOm0opwzrG3YckaSjUpCM8uPn1+AO3/jE2LCjiCQllYJkjDXb9vLE4lq+cOZIyvoVhB1HJCmpFCRj/MefVlGYG+Wr52mPI5FDCbQUzOxiM1ttZtVmdudBnv+6ma0ws7fN7EUzGxlkHslcr767gxdXbedvzz2JEwpzwo4jkrQCKwUzywLuAS4BxgPXmdn4bqstASrd/VTgCeD7QeWRzNXS3sFdf1hGef8CvnR2edhxRJJakCOFKUC1u69191ZgFnBl1xXc/SV3b0w8nA8MDzCPZKj/+sta1u7Yz91XnkJedlbYcUSSWpClMAyo6fK4NrHsUG4G/nSwJ8xshpktMrNFdXV1PRhR0l1NfSP/b141n/zIED42ZkDYcUSSXlJMNJvZDUAl8IODPe/u97p7pbtXDhigP2w5Ou7Od+YsJxox/umy7lsuReRggiyFTUBZl8fDE8vex8wuAL4NXOHuLQHmkQzz/IptzFu1na9dOIbBffPCjiOSEoIshSqgwsxGmVkOcC0wp+sKZjYJ+DXxQtgeYBbJMHub2/juf69g3OAivnhWedhxRFJGYKXg7u3ArcCzwErgMXdfbmZ3m9kVidV+APQBHjezpWY25xA/TuSY/Mv/rGDLnib+9VMfITsrKbaSiqSEaJA/3N3nAnO7Lbury/0Lgvz9kpmeW76VxxbV8tXzTtQV1USOkT5CSVrZsa+Fbz31DhOGFnP79DFhxxFJOYGOFER6k7tz55PvsLelnUevOY2cqD7ziBwr/dVI2nh8US0vrNzG3180ljGDisKOI5KSVAqSFlZtbeA7c5YzbXQ/bjp7VNhxRFKWSkFS3p7GNv7mwcUU5UX52bWTiEQs7EgiKUtzCpLSOmLO7bOXsHl3E7NmTGNgsQ5SE/kwNFKQlPaTF9bw8uo6vnP5BM4Y2S/sOCIpT6UgKeuZZVv5f/OqubpyONdPHRF2HJG0oFKQlFS1vp7bZy1hYlkJd195CmaaRxDpCSoFSTkrtzRw0wNVDCvJZ+YXK3WNBJEepFKQlLJh536+MHMhfXKjPPjXU+nfJzfsSCJpRaUgKWN7QzOfv38hbR0xHrx5CsNK8sOOJJJ2VAqSEmrqG7n612+wY18Lv7lxMicN1BHLIkHQcQqS9N7dtpcb7l9AU2sHD948lUkjdOZTkaCoFCSpLa3ZzY2/WUh2VoTHvnwm4wYXhx1JJK2pFCRpzVu1jdseWUK/Pjk8dPNURvYvDDuSSNpTKUjSicWcn817l5+88C4ThhYz88bJDNLpK0R6hUpBksqepja+Nnsp81Zt5zOnD+dfP3WKjkMQ6UUqBUkab27cxddmL2XTrib+5coJ3DBtpI5UFullKgUJXXNbBz96fg33vbKWIX3zmTVjGpXlOrmdSBhUChKqxRvq+eYTb7O2bj+fmzqCf7j0ZPrk6mUpEhb99UkoNu1u4gfPrOLppZsZVpLPQzdP5ZyK0rBjiWQ8lYL0qn0t7fzy5Wrue2UdDvztuSfyt+edpNGBSJLQX6L0ivr9rfz29fX89o317G5s46rThvLNi8fp/EUiSUalIIGqqW/k/lfXMbuqhqa2Di44eRC3nX8SE8tKwo4mIgehUpAe19zWwXMrtjG7aiOvVe8kGjGumjSMv/nYaCoG6UR2IslMpSA9oq0jxuvv7eSZZVv407Kt7G5sY1hJPl+/cAyfrRzOkL7aTCSSClQKctx27Gvh9fd28vLq7bywYhsNze0U5mQx/eRBXF1Zxlkn9icS0cFnIqlEpSBHbXtDM29u3M3iDfW8Vr2TFVsaAOibn82F4wdzySmDOaeiVKelEElhKgX5gFjMqdnVyOqte1m9dS+rtu1l6cbdbNrdBEBOVoTTR5bwzYvGcs5JpZwyrC9ZGhGIpIVAS8HMLgZ+CmQB97n797o9nwv8DjgD2Alc4+7rg8wkcbGYs21vMzX1TdTUN7KxvpGaXY28V7efd7ftpbG1o3Pdsn75nFZWwpfOLmfSiBM4ZVgxuVGNBkTSUWClYGZZwD3AhUAtUGVmc9x9RZfVbgZ2uftJZnYt8B/ANUFlSmfuTkt7jIamNhqa29jT1Ebd3lZ27Guhbm8LdQduu3y1dsQ6v98MhhTnUV5ayDWTyxg7qIixg4sYM6iIQh1YJpIxgvxrnwJUu/taADObBVwJdC2FK4F/Ttx/Avi5mZm7e4C5epy7E3OIueOJ246Y09YRo60jftve4bTFYp33Ww8s64h1rtfeEaM1cb+prYOm1naaWmP/e7+tg6a2WOf9fS0d7E2UQENT+/ve5Lsyg34FOQwoymVAUS6jSwsZUJxL2QkFlPUrYES/AoaW5OnTv4gEWgrDgJouj2uBqYdax93bzWwP0B/Y0dNhHquq4d5X1r7vjTvmTiz2/jf1mB947DjxzSze5bkPfH8v1FdOVoT8nCzys7Ped1ucF6XshHyK87MpzsumOD+auM2mOC9KaZ9cBhbl0q8wh2hWJPigIpLyUmK7gJnNAGYAjBgx4rh+xgmFOYwdVIQZRMyIJG6ty/1IhM7HRuLW7H/Xj9hhv9/433WyzMjOipCdFb+Ndrkff2zkZEWIRozsaITsSITsqBGNRMjJipCXE6EgJ0peNKI3dBHpNUGWwiagrMvj4YllB1un1syiQF/iE87v4+73AvcCVFZWHtdn8wvHD+LC8YOO51tFRDJGkB9Bq4AKMxtlZjnAtcCcbuvMAb6YuP9XwLxUm08QEUkngY0UEnMEtwLPEt8ldaa7Lzezu4FF7j4HuB940MyqgXrixSEiIiEJdE7B3ecCc7stu6vL/Wbgs0FmEBGRo6cZTBER6aRSEBGRTioFERHppFIQEZFOKgUREelkqXZYgJnVARuO89tLCeAUGj0kWbMp17FJ1lyQvNmU69gdT7aR7j7gSCulXCl8GGa2yN0rw85xMMmaTbmOTbLmguTNplzHLshs2nwkIiKdVAoiItIp00rh3rADHEayZlOuY5OsuSB5synXsQssW0bNKYiIyOFl2khBREQOI+NKwcxOM7P5ZrbUzBaZ2ZSwMx1gZreZ2SozW25m3w87T3dm9g0zczMrDTsLgJn9IPHv9baZ/d7MSkLOc7GZrTazajO7M8wsB5hZmZm9ZGYrEq+r28PO1JWZZZnZEjP7n7CzdGVmJWb2ROL1tdLMzgw7E4CZfS3x/7jMzB41s7ye/h0ZVwrA94HvuvtpwF2Jx6Ezs/OIX7N6ortPAP4z5EjvY2ZlwCeAjWFn6eJ54BR3PxVYA3wrrCBmlgXcA1wCjAeuM7PxYeXpoh34hruPB6YBX02SXAfcDqwMO8RB/BR4xt3HARNJgoxmNgz4O6DS3U8hfkmCHr/cQCaWggPFift9gc0hZunqK8D33L0FwN23h5ynux8Df0/83y8puPtz7t6eeDif+NX9wjIFqHb3te7eCswiXvKhcvct7v5m4v5e4m9uw8JNFWdmw4FPAveFnaUrM+sLfIz49V5w91Z33x1uqk5RID9xpcoCAnj/ysRSuAP4gZnVEP80Htqny27GAB81swVm9mczmxx2oAPM7Epgk7u/FXaWw7gJ+FOIv38YUNPlcS1J8uZ7gJmVA5OABeEm6fQT4h80YmEH6WYUUAf8JrFp6z4zKww7lLtvIv6etRHYAuxx9+d6+vcEepGdsJjZC8Dggzz1bWA68DV3f9LMrib+aeCCJMgVBfoRH+JPBh4zs9G9dXnSI2T7B+Kbjnrd4XK5+x8S63yb+GaSh3szWyoxsz7Ak8Ad7t6QBHkuA7a7+2IzOzfsPN1EgdOB29x9gZn9FLgT+KcwQ5nZCcRHn6OA3cDjZnaDuz/Uk78nLUvB3Q/5Jm9mvyO+HRPgcXpx6HqEXF8BnkqUwEIzixE/v0ldmNnM7CPEX4RvmRnEN9G8aWZT3H1rWLm65LsRuAyYHvL1vTcBZV0eD08sC52ZZRMvhIfd/amw8yScDVxhZpcCeUCxmT3k7jeEnAvio7xadz8wonqCeCmE7QJgnbvXAZjZU8BZQI+WQiZuPtoMfDxx/3zg3RCzdPU0cB6AmY0BckiCk3G5+zvuPtDdy929nPgfzOm9UQhHYmYXE9/8cIW0F/V9AAABoklEQVS7N4YcpwqoMLNRZpZDfAJwTsiZsHiT3w+sdPcfhZ3nAHf/lrsPT7ymrgXmJUkhkHht15jZ2MSi6cCKECMdsBGYZmYFif/X6QQwAZ6WI4UjuAX4aWKiphmYEXKeA2YCM81sGdAKfDHkT76p4OdALvB8YhQz392/HEYQd283s1uBZ4nvFTLT3ZeHkaWbs4HPA++Y2dLEsn9IXD9dDu024OFEwa8FvhRyHhKbsp4A3iS+uXQJARzZrCOaRUSkUyZuPhIRkUNQKYiISCeVgoiIdFIpiIhIJ5WCiIh0UimIiEgnlYKIiHRSKYh8SGY2OXFNhzwzK0yc7/6UsHOJHA8dvCbSA8zs/xI/h08+8fPm/HvIkUSOi0pBpAckTodQRfzUKWe5e0fIkUSOizYfifSM/kAfoIj4iEEkJWmkINIDzGwO8autjQKGuPutIUcSOS6ZeJZUkR5lZl8A2tz9kcS1ml83s/PdfV7Y2USOlUYKIiLSSXMKIiLSSaUgIiKdVAoiItJJpSAiIp1UCiIi0kmlICIinVQKIiLSSaUgIiKd/j8e5xWAs9qDLQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline \n",
    "x = np.arange(-8, 8, 0.1)\n",
    "f = 1 / (1 + np.exp(-x))\n",
    "plt.plot(x, f)\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('f(x)')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bias and weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEKCAYAAAAW8vJGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XdgFGX6wPHvm2w66QkthdBJaIEEkCqK3lnhPBsgiO2wnHeWa5z6U0+9O2xnP8uJAoqHiAqoeIiCiqhAaCGFEmoa6ZXUzb6/PyZAAiF1N7sbno83tzsz78w8gbDPvvO+875Ka40QQghxkou9AxBCCOFYJDEIIYRoRBKDEEKIRiQxCCGEaEQSgxBCiEYkMQghhGhEEoMQQohGJDEIIYRoRBKDEEKIRkz2DqA9QkJCdFRUlL3DEEIIp7J9+/Z8rXVoS+WcMjFERUWRkJBg7zCEEMKpKKWOtqac3EoSQgjRiCQGIYQQjUhiEEII0YhTtjE0pba2loyMDKqqquwdikPz9PQkPDwcNzc3e4cihHBQXSYxZGRk4OvrS1RUFEope4fjkLTWFBQUkJGRQd++fe0djhDCQdn0VpJS6h2lVK5SKukc+5VS6mWlVJpSKlEpNbq916qqqiI4OFiSQjOUUgQHB0utSgjRLFu3MSwGLmtm/+XAwPplPvB6Ry4mSaFl8mckhGiJTW8laa2/V0pFNVNkBrBUG/OL/qyUClBK9dJaZ9syLiFEx2itqbXUUmmupNJcSYW5gkpzJVXmKswWM3WWOsy66dc6XYfZYsZsMaPRaK0bvTa8xqn/6qcgPrm/4TEnt2s0xv8an6djP6gFLHWnF11nbNO6/tViXN1Sd3obDfad2tZwXdeXqX81fqAG26h/T5P7rxhxK/36XGidn+8c7N3GEAakN1jPqN92VmJQSs3HqFUQGRnZKcE5kn/+858sWrQIV1dXXn75ZX75y1+eVeaWW27hu+++w9/fH4DFixcTGxvb2aEKJ1ZSXcKx0mOkl6VzvOI4hZWFFFYVUlhdeOp9eW05VeYq6nSdvcN1KEpbKRm1IDozussnhlbTWr8FvAUQHx/fOX8DDiIlJYXly5eTnJxMVlYWl1xyCfv378fV1fWsss8++yzXXXedHaIUzkRrTUZ5Bkn5SSTnJ5NUkMT+ov2U1ZQ1Kufh6kGwZzCBnoGEeIUwMHAgfu5+eJm8mlw8TZ6YXEy4ubjhqlxxdXHFVblicjFhcjGden9ynwsup25vKqU4+d+pdYuGE7mosmxUWTaUHkeVZaHKjqPKc6CiEFVRCJWFqLoaFJxazuLqAe4+4NEN3E8u3mDyApM7mDzB1R1MHme89zCONbmDixu4mOoX1/rFBKr+1cXljPUmyiiX+kXVLy5GxKe2nWu9fnHzsd0vRj17J4ZMIKLBenj9Nqfz7LPP4uHhwe9//3seeOABdu/ezYYNG9iwYQOLFi1i2bJl7T736tWrmTlzJh4eHvTt25cBAwawdetWxo8fb8WfQHR1tZZadubs5NuMb/k2/VvSy4zKuruLO0OChnB51OVE+kUS4RtBpG8kvbr1wtvk3TntUlWlcDwZ8vdD4UEoOAiFh6DwMJgrG5d1dQffXsYS2BfC4sA7CLyDwav+1TsIPP3rP/x9jFeTu+1/ji7C3olhDXCvUmo5MA4osUb7wt8+SyYlq7TDwTUU09uPx64ees79kydP5vnnn+f3v/89CQkJVFdXU1tby6ZNm5gyZcpZ5R944AE2btx41vaZM2eyYMGCRtsyMzO54IILTq2Hh4eTmdl0/nz44Yd54oknmDZtGgsXLsTDw6O1P6Loog6VHOKjfR/x2aHPKKkuwd3FnbG9xnJzzM3Edo+lf0B/3Fw68bmWikLI3AHHd0N2IhxPNJLASa7uEBgFQf2h/8XGe/8I8OttLN7BxjfnLqrGbCG3rIrjJVVkl1SRU2q8GuuVPHRFNPFRQTaNwaaJQSn1X2AqEKKUygAeA9wAtNZvAGuBK4A0oAK41Zbx2FJcXBzbt2+ntLQUDw8PRo8eTUJCAps2beLll18+q/wLL7xg9Rj++c9/0rNnT2pqapg/fz5PP/00jz76qNWvI5xDakEqbya+yTfHvsHkYmJa5DQuj7qc8b3H4+3m3XmBVBTCoW/h6I/Gkpt8el9AH+g1AkbONl5Dh4B/uHH7pYuqMVvILK7kWGGFsRSc4FhhBVnFRgLIL68+6xgvN1d6+XvS098TSyfcSLd1r6RZLezXwG+tfd3mvtnbipubG3379mXx4sVMmDCBESNGsHHjRtLS0oiOjj6rfFtqDGFhYaSnn26jz8jIICws7Kxje/XqBYCHhwe33norzz33XEd/LOGECioLeHnny3x64FO6uXXjrpF3MXPwTIK9gjsnAK0hJxkOrIP96yBjm9Ejx80HIsfBsGsgYhz0HAFeAZ0TUyfTWpNTWk1abjkHcstIyy3nUJ6RALJLKht9uHuYXIgI8iYswIthYX708POkl79n/asXPf098fM0dWpXc3vfSupSJk+ezHPPPcc777zD8OHDefDBB4mLi2vyL7QtNYbp06cze/ZsHnzwQbKysjhw4ABjx449q1x2dja9evVCa82qVasYNmxYh34e4Xy+OfoNj/30GCdqTjBv6Dzmj5iPr7tv51y88BAkfgR7PoKCA8a2XrEw5U8w4FLoHQuuXW8oluKKGpKzSknOKmF/TjkHcss5lFtOWbX5VBk/TxP9u3djbN8gIoK8iaxf+gR7E9rNAxcXx7o1JonBiiZPnszf//53xo8fj4+PD56enkyePLnD5x06dCg33HADMTExmEwmXnvttVM9kq644grefvttevfuzU033UReXh5aa2JjY3njjTc6fG3hHGottSzcspAV+1cQExzDPyb9g/4B/W1/YXM1JH0CCYuMmgEKoibB+N/C4MvBt6ftY+hEuWVVJGeWkpRZQlJWCUmZpWQWn24c7+7rwYDu3bhmdBgDunc7tYR283Cqh0uV7qS+t9YUHx+vz5yoJzU1tclbNuJs8mfVtZTWlPLgtw+yJXsLtw69ld+N+h1utv5mXpoF2xbB9sVQkQ8hg2DUHBh2rdFG0AXU1llIzS5l+9EiEo4WseNoEdklp4eT6Rviw9DefgwL82dYb3+G9vYj0Mexez4ppbZrreNbKic1BiGcWHFVMXd8dQcHSw7y1MSnmDFghm0vWJoFm56H7UvAYjZqBWPnQ7+pTt9TqMZsYVd6MZvT8tlyuIDd6SVU1hoP8YUFeBEfFURsRADDw/yJ7uWLr2fXuy12kiQGIZxUSXUJ89fP53DJYV67+DUmhE2w3cXKc+GHF4xagq6DUXNh4n0Q5Lyj9FosmpTsUjan5bP5YAHbDhdSWVuHUjC0tx83jokgPiqQuD6B9PL3sne4nUoSgxBOqLaulvs23kdacRovXfSS7ZJCnRm2vQ0b/w41J2DkLLjwT8azBU6oosbM5rQCvknNYcPeXHLLjK6hA7p344b4cMb3D2F8v2D8vbtubaA1JDEI4WS01vxj6z/YnrOdhZMXMjm84x0cmpS+Db54AI7vgf7T4PKnIWSgba5lQwXl1fwv+Thfp+Tw48ECqs0WunmYuHBQKBcN6c7kgSH08PO0d5gORRKDEE7m4wMfs3L/Sm4bdhtX9rvS+heorYT1j8LWt8C3N1y/BGJmOFUbQnFFDeuSj/N5YjY/HiygzqLpE+zNTeP6MC26O2OignA3yczG5yKJQQgncrT0KM9se4YLel3A70f93voXyE2FlbdBbgqMuwsufgQ8Ouk5iA6qqq3jf0nHWbUrkx8O5GOuTwZ3XdiPq0b0ZkhPX6fqMmpPkhicQEFBAddddx3btm3jlltu4dVXX22yXGFhITfeeCNHjhwhKiqKFStWEBgY2MnRClsxW8w89MNDmFxMPDXxKVytOWyE1pDwDqx7yEgEN30MAy+x3vltRGtNUmYpHyYcY/WuLMqqzIQFeHH7pL5cNaI3w8L8JBm0gyQGJ+Dp6cmTTz5JUlISSUlNzpIKwMKFC5k2bRoLFixg4cKFLFy4kKeffroTIxW2tCx1GYl5iTw9+Wl6+PSw3onN1bDmd5D4odGWcM0b0K279c5vAyWVtXy6I4MPEzJIzS7Fw+TC5cN6csOYCC7oG+xwTxI7G0kMVmLLYbd9fHyYNGkSaWlpzZZbvXo13377LQDz5s1j6tSpkhi6iPzKfF7f/TpTwqdwRb8rrHfiikJYfhMc+xEuehgm/9GYU8BBHcorZ/GPR1i5PYOKmjqGh/nz5K+GMX1kb/y9zu+eRNbUNRPDlwuMnhTW1HM4XL7wnLttOex2a+Xk5JwaSK9nz57k5OS06zzC8by4/UVq6mr4y5i/WO+k+WnwwfVQkgnXLoLhjjnBk9aaH9LyeXfzETbszcXd1YWrR/bm1olRDAvzt3d4XVLXTAx24AjDbjeklJJ7q11EUn4Sqw+u5rZhtxHpZ6VpbTO2w7JrjdnB5n1mjHrqYOosms8Ts3htYxr7c8oJ6ebO/ZcM5KZxfQj1lXlGbKlrJoZmvtnbii2H3W6tHj16nBphNTs7m+7dHfs+sWidV3e9SqBHIPNHzLfOCTO2w3u/MmY5m/spBPWzznmtxFxnYfUuIyEcyj/B4B6+PH/9SK4a2QsPU9edp8GRdM3EYCe2Gna7taZPn86SJUtYsGABS5YsYcYMG4+bI2xuT94eNmdu5v7R9+Njjbl+M7fDe9cYSeGWLxxqwLvaOguf7sjk1Y1pHCusIKaXH2/MieMXMT2kMbmTSWKwIlsNuw0QFRVFaWkpNTU1rFq1iq+++oqYmBjuuOMO7rrrLuLj41mwYAE33HADixYtok+fPqxYscIq1xb28/ru1wnwCGDWkGbnvGqdzO2w9Bpjcpx5nztMUtBa83liNs+u28exwgpGhPvz6FXxTIvuLrdD7UQSgxVNmzaN2traU+v79++32rmPHDnS5Pa333771Pvg4GC++eYbq11T2FdyQTKbMjdx3+j7Oj4VZ26qUVPw8odbPoeACOsE2UHbjhTy9y9S2ZVeTHQvP969ZQxTB4dKQrAzSQxCOKhlKcvwcfNh5uCZHTtReS4suwFMnkZNIcBKDdgdcCivnIVf7uWrlBx6+nny3PUjuWZUGK5yy8ghSGIQwgHlV+bz5ZEvuWHQDXRz79b+E9VWwn9nwYk8uHUtBPaxXpDtUFZVy7/W7+e9n47iYXLhT78czG0T++LlLo3KjkQSgxAOaOX+lZgt5o61LVgs8OmdRtvCje9B2GjrBdhGWmu+TDrO3z5LJresmlljI3ngkkHS7dRBSWIQwsHU1tWyYt8KJoZNJMo/qv0n2vAkpKyGS5+E6KutFl9bHSuo4P9WJ/Hd/jyG9vbjzbnxxEYE2C0e0TJJDEI4mO8zvievMo/Hhzze/pPsXQs//AtGz4MJv7NabG1RY7bwn02HePmbA5hcFI9eFcPN4/tgcnXcITeEQRKDEA5m9cHVhHiFMKF3O2dlK06HVXdDr5FwxbN2mUdh7/FSHvhwN6nZpVwxvCePXjWUnv4yGY6zkNTtBNavX09cXBzDhw8nLi6ODRs2NFmusLCQSy+9lIEDB3LppZdSVFTUyZGKjiqsKmRTxiau6ncVJpd2fG+rq4WPbwdLHVz3Lpg69x5+nUXzxncHmf7KZvLKqnhrbhz/vilOkoKTkcTgBEJCQvjss8/Ys2cPS5YsYe7cuU2WOzns9oEDB5g2bRoLF3b+0CCiY748/CVmbWZ6/+ntO8HGf0D6Frj6RQjub93gWnC04AQz3/qJhV/u5eIh3Vl3/xR+MbRnp8YgrEMSg5U8++yzpwbLe+CBB7j44osB2LBhAzfddFOHzj1q1Ch69+4NwNChQ6msrKS6uvqscqtXr2bevHmAMez2qlWrOnRd0fnWHFxDdFA0AwPbMbdy2ten2xU6caRUrTUfbDnG5S9tYu/xMl64cSSvzxlNcDfpceSsumQbw9Nbn2Zv4V6rnnNI0BD+MvbcQx531rDbH3/8MaNHj8bD4+x/dDLstnM7XHKYlIKU9g2tXVkEq+6B0Gi4rPNqimVVtSz4eA9f7Mlm0oAQnrluBL0DvDrt+sI2umRisIfOGHY7OTmZv/zlL3z11VctlpVht53P+qPrAbi0z6VtP/irR+BEPsxeAe4dHD6jlVKySrln2XbSiypZcPkQ5k/uJ4PddRFdMjE0983eVmw97HZGRgbXXHMNS5cupX//pu8dy7Dbzu3ro18zMnRk26ftPLgRdr4Pkx6A3rG2Ca4BrTXLt6Xz2JpkAr3dWD7/AsZEBdn8uqLz2DwxKKUuA14CXIG3tdYLz9gfCSwBAurLLNBar7V1XLZgq2G3i4uLufLKK1m4cCETJ048ZzkZdtt5pZelk1qYyh/j/9i2A6vL4bPfQ/AAuLB983i0xYlqM4+sSuLTnZlMHhjCCzfGEiJtCV2OTRuflVKuwGvA5UAMMEspFXNGsUeAFVrrUcBM4N+2jMmWJk+eTHZ2NuPHj6dHjx5WG3b71VdfJS0tjSeeeILY2FhiY2PJzc0F4I477iAhIQGABQsWsH79egYOHMjXX3/d7gl/ROf7+ujXAEyLnNa2Azc8BcXHYPqr4GbbLqHHCiq45t+bWbUrkwcvHcTiW8dKUuiibF1jGAukaa0PASillgMzgJQGZTTgV//eH8iycUw2Y6thtx955BEeeeSRJvfJsNtdw9dHvyY6KJpw3zbMkZC+Fba8AWN+A33G2y444OdDBdz9/nYsGt67bRyTBobY9HrCvmzdXTUMSG+wnlG/raHHgTlKqQxgLWCf5/eFsJO8ijwS8xPbVluw1MHnD4Jfb7jkMdsFByzfeow5b28hyMed1b+dKEnhPOAIzzHMAhZrrcOBK4D3lFJnxaWUmq+USlBKJeTl5XV6kELYyg+ZPwBwYcSFrT9ox1LI2QO/eBI8fG0Sl7nOwhOfpbDgkz1MGBDCJ/dMJCrECtOLCodn68SQCTScKiq8fltDtwMrALTWPwGewFlfSbTWb2mt47XW8aGhoU1eTGttjZi7NPkzcjw/ZP5AqFcogwMHt+6AqhKjbSFyPAz9tU1iKq2q5fYlCbyz+TC3TozinXnx+Hu52eRawvHYOjFsAwYqpfoqpdwxGpfXnFHmGDANQCkVjZEY2lwl8PT0pKCgQD74mqG1pqCgAE9PGbfGUdRaavkp6ycmh09u/XMn3z0DFQXGg2w2eFbleEkV17/+E5vT8vnHNcN57OqhMiLqecamjc9aa7NS6l5gHUZX1He01slKqSeABK31GuAPwH+UUg9gNETfotvx6R4eHk5GRgZym6l5np6ehIc7xiTwAnbn7qastoxJYZNad0DBQdjyJoy6ySbPLKTlljPvna0UV9Sw+Nax0p5wnrL5cwz1zySsPWPbow3epwDn7pzfSicfMBPCmfyQ+QMmZeKCXhe07oB1DxtzN1/8aMtl22jnsSJuW7wNVxfFh3eOZ1iYv9WvIZyD1A+FsKNNmZsY1WMUvu6taEA+uBH2fwlT/gi+bXw6ugUb9+Uy+z9b8PV04+O7J0hSOM9JYhDCTvIr89lftL91E/JoDd/8Dfwj4YK7rRrHJzsy+M2SBPqF+vDx3RPoEyw9j853XXKsJCGcQcJx44n1Vt1G2vsFZO2EGa9ZdfKdxZsP8/hnKUzoH8ybc+Pw9ZSeR0ISgxB2s+X4FnzdfBkSNKT5ghYLbPy7MR7SiJlWu/5/vj/E39em8ouYHrwyexQeJlernVs4N0kMQtjJluwtxPWMa3kKz+RPIDcFrl0Ertb5J/vvb9N45n/7uHJ4L16cGYubdEcVDchvgxB2kFWeRXpZOuN6jmu+YJ3ZmK6z+1CrPcz28jcHeOZ/+5gR25uXJCmIJkiNQQg72Hp8KwBje41tvmDicig8CDM/AJeOfYBrrXlh/X5e3pDGr0eH8ex1I3GViXVEEyQxCGEHW7O3EuQZxMCAZuZ2NlfDt09D79Ew+IoOXU9rzTPr9vH6twe5MT6Cf/x6uCQFcU6SGIToZFprth7fypieY5ofBmP3f6HkGFz9QoeHvnjh6wO8/u1BZo+L5KkZw2QKTtEsubkoRCfLOpFFTkUOo7uPPnchSx1sfgl6j4L+bZy85wxvfHeQl785wI3xEZIURKtIYhCik+3I2QFAXI+4cxdKWQWFh2DSgx2qLbz30xEWfrmXq0f25h+/Hi5JQbSKJAYhOtnO3J10c+vGgIABTRfQGn54AYIHwpCr2n2dldsz+L/VyVwS3YN/3SANzaL1JDEI0cl25OxgZPeRuLqc44GytG/g+B6YdH+7eyJ9kZjNn1fuZvLAEF6dPUq6pIo2kd8WITpRcVUxB0sOEte9mdtIP/wL/MJg+A3tusaGvTnct3wncX0CeXNuHJ5u8kSzaBtJDEJ0ol15uwAY1X1U0wWObYGjm2H8vWByb/P5E44Ucvf7O4ju5ceiW8bg7S4dD0XbSWIQohPtyNmBycXEsJBhTRf44QXwCoK4eW0+d1puGbcvSaB3gBeLbx2DnwyIJ9pJEoMQnWhn7k6GBg/F09TE9Kp5+4z5FsbdCe5tG/r6eEkVNy/airvJhaW3jSW4m/VGYBXnH0kMQnSSWkstqYWpjAgd0XSBn183Zmcbc0ebzltSWcst726ltMrMu7eMISLI2wrRivOZJAYhOsmBogNU11UzIqSJxFBRCLuXw/Drwaf18yxX1dYxf2kCB/PKeXNunMy8JqxCWqaE6CRJ+UkATbcvbF8M5so2zc5WZ9E8uGIXWw4X8tLMWCYOaH1CEaI5UmMQopPsyd9DoEcgYd3CGu+oq4Wt/4G+F0KPoa0+31NfpLB2z3EeuTKaGbFhLR8gRCtJYhCikyTlJzEsZNjZA+elroGyLLjgnlafa8mPR3h38xFum9iXOyb3s3Kk4nwniUGITlBeU87B4oMMDx1+9s6fX4egfjDwF60618a9ufztM2Ooi4evjLZypEJIYhCiU6QUpKDRDA85IzGkb4OMbTDu7lYNf5GaXcq9HxgPsL00M1bGPxI2IYlBiE6wJ38PAMOCz2h43vI6ePhB7KwWz5FbWsXti7fRzdPEonlj8PGQviPCNiQxCNEJkvKTiPSNJMAz4PTGsuOQshpGzQUP32aPr6yp446lCRRX1rJo3hh6+jfxgJwQViKJQYhOkJifeHY31R1LwWKGMbc3e6zFonngw13sySzh5Zmj5FkFYXOSGISwsZwTOeRW5DZuX6gzQ8K70P9iCO7f7PHPr9/H/5KP88iVMVwS08PG0QohiUEIm0sqaOLBtv3/M7qoxjdfW/hsdxavbTzIzDER3DYxyoZRCnGaJAYhbGxP3h5MykR0cIOupdveNuZcGHTZOY9LyizhTyt3E98nkCdmNPH8gxA2IolBCBtLyk9iUNAgPFzrRzwtOAiHNkLcreDadM+ivLJq5i9NIMjbndfnxOFukn+qovPY/LdNKXWZUmqfUipNKbXgHGVuUEqlKKWSlVIf2DomITqLRVtIKkhq3L6Q8A64mGD0zU0eU2O2cM+y7RRW1PDWzfGE+soQ2qJz2bQjtFLKFXgNuBTIALYppdZorVMalBkI/BWYqLUuUkp1t2VMQnSmIyVHOFF74nRiqK2Ene9D9NXge3ZDstaax9Yks+1IEa/Mkh5Iwj5sXWMYC6RprQ9prWuA5cCMM8r8BnhNa10EoLXOtXFMQnSaxPxEgNOJIekTqCo+55wL7/98lP9uPcY9U/tz9cjenRWmEI3YOjGEAekN1jPqtzU0CBiklNqslPpZKdVka5xSar5SKkEplZCXl2ejcIWwrqT8JHzcfIjyjzI2JLwDIYOhz8Szyv50sIC/fZbCxUO684dfDO7cQIVowBFatEzAQGAqMAv4j1Iq4MxCWuu3tNbxWuv40NDQTg5RiPZJLUglOigaF+UCOcmQmQBxt8AZPYzSCyu4Z9l2+gR786KMgSTszNaJIROIaLAeXr+toQxgjda6Vmt9GNiPkSiEcGpmi5l9RftOd1PdsRRc3WHkzEblKmrM/GZpAnUWzdvzxuDn6WaHaIU4zdaJYRswUCnVVynlDswE1pxRZhVGbQGlVAjGraVDNo5LCJs7XHKY6rpqYoJjoLbKmLoz+mrwDjpVRmvNXz/Zw76cMl6eNYq+IT52jFgIg00Tg9baDNwLrANSgRVa62Sl1BNKqen1xdYBBUqpFGAj8CetdYEt4xKiM6QWpgIQExQDez83Gp3P6KK65McjrN6VxR8uHcTUwdIhTzgGm4/bq7VeC6w9Y9ujDd5r4MH6RYguI7UgFS+TF338+sCO+yCgD0RNObU/4UghT32RyiXR3bln6gA7RipEY61ODEopF2Ak0BuoBJKka6kQ55ZSkMLgwMG4Fh+Fw9/DxY+cmownt6yKe5btIDzQi+dviMVFGpuFA2kxMSil+gN/AS4BDgB5gCdGF9MK4E1gidbaYstAhXAmFm1hb+FeZgyYYTzQplwg9iYAauss3LtsJ6VVtSy9fSz+XtLYLBxLa2oMTwGvA3fW3/Y5pf4p5dnAXGCJ9cMTwjkdKz1GhbmC6MDBsOkhYz5nP+OBtYVf7mXrkUJevDGWIT397BypEGdrMTForc8552D9raQXrRqREF3AqYbnsiIoPw6j5wGwZncWi344zC0TovjVqDOf9RTCMbS6V5JS6kmllKnBup9S6l3bhCWEc0stSMXNxY1+e9dDt54w8BfszynjLysTiesTyENXRLd8EiHspC3dVU3AFqXUCKXUpRjPKGy3TVhCOLeUghQG+fXFLe0riJ1Naa3mrve24+Nh4t83jZZhtIVDa3WvJK31X5VSXwNbgCJgitY6zWaRCeGktNakFKbwS49eoC1YYufwxxW7OVpYwQd3jKOHn6e9QxSiWW25lTQFeBl4AvgWeEUpJcM/CnGGzPJMymrKiM45AFGTeSNJ81VKDn+9fAjj+gXbOzwhWtSWB9yeA64/OZeCUurXwAZgiC0CE8JZnWp4LspmX//7eW7dPq4a0YvbJ/W1c2RCtE5bEsN4rXXdyRWt9SdKqe9sEJMQTi21IBUTiv7Km4t+7kH/0G48fe0ImbNZOI0WbyUppeYopVwaJoWTtNYFSqno1SVbAAAgAElEQVT+SqlJtglPCOeTkreb/jU1rFdTKK8z8cbcOHw8bD76jBBW05rf1mBgp1JqO0YvpJNPPg8ALgTygSbnchbifKO1JjVvD1Oqq/l3yQSeu2kk/UO72TssIdqkNQ+4vaSUehW4GJgIjMAYKykVmKu1PmbbEIVwHrkVORTWVeJX5cdFF17EZcN62jskIdqsVfVbrXWdUmqT1nq9rQMSwplt2PYRAJUek3jo0kF2jkaI9mnLjc8kpVQOsKl++UFrXWKbsIRwPkUnavh598cof81tN/4Vk6s8xCacU6t/c7XWAzDmZN4DXAnsVkrtslVgQjiTOovmL//9kVq3TCKVN+Hdw+0dkhDt1pYH3MIx2hgmA6OAZOBDG8UlhFN56ev9BB7+nH0eJob1GGnvcITokLbcSjqGMT7SP7TWd9koHiGczjepOby8IY1VwZtYazIRHSG9t4Vza8tN0FHAUmC2UuonpdRSpdTtNopLCKdwtOAED3y4iyu6F1JuOQJATHCMfYMSooPa0sawG2MynncxhsK4EHi02YOE6MIqa+q46/0dKKVY2HcXqZ5eAAwOGmznyITomLbM+ZwAeAA/YvRKmqK1PmqrwIRwZFprHl61h73HS1k8dwR+n80nNSySCO9u+LnLrGzCubWljeFyrXXeuXYqpeZprWV6T3FeWLblGJ/syOT+SwZyYd0WqCwixRTF0CCZgEc4v7bcSjpnUqh3XwdjEcIp7DxWxN8+S2bq4FB+f/FA2LGEksBIMqsLiA6WxCCcnzWfwJGhI0WXl19ezT3LdtDDz5MXb4zFpfgwHP6evUMuBaThWXQN1kwM2ornEsLhmOss/O6DnRSeqOGNOXEEeLvDjqWgXEkOjgQgWm4liS7AmmMBS41BdGnPfbWfnw4V8Ox1IxgW5g/mGtj5Pgy6jJQTGYR1CyPQM9DeYQrRYW3pleQBXAtENTxOa/1E/dvNVo1MCAfyv6TjvPHdQWaPi+T6+Ahj4761cCIP4m4hec8LchtJdBltuZW0GpgBmIETDRYAtNb3Wjc0IRzDobxy/vjRbkaG+/PY1Q0+/LcvBv8ISiLiySjPYGjwULvFKIQ1teVWUrjW+jKbRSKEAyqvNnPX+9txc1X8e04cHiZXY0fhITi0ES56mJSifYA0PIuuoy01hh+VUsNtFokQDsZi0fxhxS7Scst5ZdZowgK8Tu+sb3Rm1BySC5IBSQyi62jNnM97lFKJwCRgh1Jqn1IqscH2lo6/rP6YNKXUOacAVUpdq5TSSqn4tv0IQtjGqxvTWJecw0NXRDNpYMjpHQ0anfHrTUpBChG+Efh7+NsvWCGsqDW3kq5q78mVUq7Aa8ClQAawTSm1RmudckY5X4wH5La091pCWNP6lBz+tX4/vx4Vxu2T+jbe2aDRGSA5P5kRoSM6P0ghbKTFGoPW+mhzSwuHjwXStNaHtNY1wHKMBuwzPQk8DVS1+ScQwsrScst44MNdjAj35x+/Ho5SZ/TErm90ZsA0iqqKyDqRJbeRRJdi67kHw4D0BusZ9dtOUUqNBiK01l/YOBYhWlRSWctvlm7H082FN+bE4enm2rjAyUbn0TeDiyspBUblV3okia7Emg+4tZlSygX4F3BLK8rOB+YDREZG2jYwcV6qs2juW76T9MIKPvjNBfRu2Nh8UoNGZ+BUw7OMkSS6ElvXGDKBiAbr4fXbTvIFhgHfKqWOABcAa5pqgNZav6W1jtdax4eGhtowZHG+ev6rfXy7L4/Hpw9lbN+gswuc0egMkFKQQpRfFL7uvp0crRC2Y+vEsA0YqJTqq5RyB2YCa07u1FqXaK1DtNZRWuso4GdgutY6wcZxCdHI54lZ/Pvbg8waG8FN485RIz2j0RmMGoPUFkRXY9PEoLU2A/cC64BUYIXWOlkp9YRSarotry1Ea6VklfKnjxKJ6xPI49OHnt3YfFKDRmeA/Mp8jp84Lu0LosuxeRuD1notsPaMbU1OCaq1nmrreIRoKK+smt8sTcDPy8TrN40+/WTzmfLTTj3pjItRRhqeRVdl18ZnIeypqraO+e8lUHCimo/unEB3P89zF972Nri4weh5pzalFKSgUHIrSXQ5khjEeUlrzZ9XJrLzWDFvzBnN8PBmnlquLoddy2Dor8C3x6nNyQXJ9PXvi4+bTydELETnsXXjsxAO6ZUNaazZncWffjmYy4b1ar5w4nKoLoWxdzbanJKfIg+2iS5JEoM473yemGUMdzE6jHum9m++sNaw9T/QKxbCT/eizqvII7cyV9oXRJckiUGcV3anF/OHFbuJ7xPIP5sa7uJMh7+HvL0wdj40KLsnfw8Aw0KG2TJcIexCEoM4b2QVV/KbpQmE+nrw5ty4c/dAamjrW+AVBMOubbQ5MS8Rk4tJGp5FlySJQZwXSiprueXdrVTW1LFo3hiCu3m0fFDxMeOhtrh54Na4x1JifiLRQdF4uLbiPEI4GUkMosurNtdx53sJHM4/wZtz4xjcs5XDV2xbZLzG395os9liJik/ieEhMm+V6Jqku6ro0iwWo1vqz4cKefHGWCYMCGn5IICaE7BjCQy+AgIiGu06WHyQSnOlzMEguiypMYgu7dmv9rF6VxZ/vmwwvxoV1vIBJ+1cBpVFMOF3Z+3anbcbQBKD6LIkMYgu672fjvD6tweZc0Ekd1/YQrfUhix18PNrED4GIsadtTsxL5EgzyDCu4VbL1ghHIgkBtElrU/J4bE1yVwS3Z3Hr25mYLym7P0cio4YtYUmjkvMT2REyIi2nVMIJyKJQXQ5244U8rv/7mB4eAAvzxqFybUNv+Zaw+aXITAKhpw93XlJdQmHSw4zPFQankXXJYlBdCnJWSXctngbvQO8eGdePN7ubexfkb4FMhNg/L2nRlFtaFfuLgBGdR9ljXCFcEiSGESXcSivnJsXbcXXw8T7t49r3bMKZ/rxFfAKhNjZTe7ekbsDk4tJnngWXZokBtElZBVXMuftLQC8f8e4pudrbkl+Guz9AsbcAe5Nj5i6I2cHMcExeJnacX4hnIQkBuH0CsqrmbNoC2VVZpbcNpZ+od3ad6KfXgVXN2NcpCZUmatIKkgirntcB6IVwvFJYhBOrbSqlnnvbiWruJJ3bh3DsLBm5lVoTkkG7HwfYm+Cbt2bLJJckIzZYpb2BdHlSWIQTutEtZnbF29jb3YZr8+JY0xUUPtPtvklQMPkB89ZZEfODkAankXXJ0NiCKd0otrMre9uY8exYl6eOYqLBjf9Lb9VSrNh+xKjwTkg8pzFduTuoL9/fwI8A9p/LSGcgNQYhNM5UW3m1sXb2H6siJdmxnLliBZmYGvJjy+DxQyTzl1bqLPUsTt3N6N6SG1BdH2SGIRTOZUUjhbx4o2xXDWid8dOWJYDCe/AyJkQ1PecxfYW7qWstowxPcZ07HpCOAFJDMJpVNQYSSHhiDFS6tUjO5gUwKgt1NXA5D80W2zLcaMr7NheYzt+TSEcnCQG4RQqaow2hYQjhbw4c5R1kkJ5nlFbGH4DBDc/yN7W7K309+9PiFcrh+0WwolJYhAOr6SylpsXbWXbkUJeuDGW6dZICgCbX4TayhZrC7V1tezI3SG1BXHekF5JwqHllVVz8ztbScst45VZozve0HxS0VFjPufYmyB0ULNFE/MTqTRXMq7n2UNwC9EVSWIQDiujqII5b28hp7SaRfPGMGVQqPVOvvHvoFzgoodaLLo1eysKRXzPeOtdXwgHJolBOKS03DLmvL2Vihoz798xjrg+gdY7efZuSFwBk+4H/5ZnddtyfAvRwdH4e7TzqWohnIy0MQiHk5hRzPVv/ESd1nx453jrJgWA9Y+BVwBMvL/FouU15ezO2824XnIbSZw/JDEIh7LpQB6z/7OFbp4mVt41nuhefta9wMENcGgjTPmzkRxasCV7C2aLmclhk60bhxAOTBKDcBgfbDnGLe9uIzzQi5V3TaBPcNNDX7ebxQLrHzWGvRhze6sO2ZS5CR83H2K7x1o3FiEcmM0Tg1LqMqXUPqVUmlJqQRP7H1RKpSilEpVS3yil+tg6JuFYLBbNP9am8tCne5gyMISVd0+gh5+n9S+0+wM4vgcufhRMLU/io7VmU+YmJvSegJuLm/XjEcJB2TQxKKVcgdeAy4EYYJZSKuaMYjuBeK31CGAl8IwtYxKOpbKmjruXbeet7w9x8/g+/OfmeLp52KBPREWhUVuIuACGXduqQ/YX7Se3IlduI4nzjq17JY0F0rTWhwCUUsuBGUDKyQJa640Nyv8MzLFxTMJB5JZV8ZslCSRmlvDoVTHcOjEKpZRtLvbN36CyGK58Hlxa933oh8wfAJgYNtE2MQnhoGydGMKA9AbrGUBz3TtuB75saodSaj4wHyAy8txDIwvnsCejhDvfS6Coopa35sZzaUwP210sfZsxrPb430LP1s/V/H3G9wwJGkJ37w4M6S2EE3KYxmel1BwgHni2qf1a67e01vFa6/jQUCs+6CQ63crtGVz7xo8AfHTXeNsmhTozfPEA+PaCqWc1cZ1TQWUBu/J2MTViqu1iE8JB2brGkAlENFgPr9/WiFLqEuBh4EKtdbWNYxJ2Ultn4anPU1jy01HG9wvm1dmjCO7WciNwh2x722hwvn4JePi2+rAN6RuwaAuXRF5iw+CEcEy2TgzbgIFKqb4YCWEmMLthAaXUKOBN4DKtda6N4xF2kl1Sye8+2EnC0SJ+M7kvf7lsCCZXG1dYSzJgw1PQfxrEzGjToeuPrCfSN5JBgc2PoyREV2TTxKC1Niul7gXWAa7AO1rrZKXUE0CC1noNxq2jbsBH9Q2Px7TW020Zl+hcG/fm8uCKXdSYLbw8a5T1RkdtjsUCq+4BbYErn4M2NGqXVJew7fg2bh56s+0aw4VwYDYfK0lrvRZYe8a2Rxu8l7p6F1VbZ+G5dft48/tDRPfy47XZo+gX2q1zLr71LTj8HVz1IgT1a9OhG9M3YtZmLu1zqY2CE8KxySB6wiYO5ZXzwIrd7E4v5qZxkfzfVTF4url2zsXz9sHXj8HAX0LcLW0+fP3R9fTy6cXQ4KHWj00IJyCJQViV1pplW47x9y9ScTe58NpsK86h0Bp1tfDJfHDzhumvtOkWEkB+ZT6bMzczb+g8uY0kzluSGITV5JZVseDjPWzYm8vkgSE8e91IevrbYGiL5nz3DGTvghveA9+2d4Nde2gtdbqO6f2lmUucvyQxiA7TWrNyewZPfZFKVW0dj18dw83jo3Bx6eRv3Ac3wKbnYOQsiGnfB/uag2sYGjyU/gHNzwEtRFcmiUF0SHphBQ99uodNB/IZExXI09eO6LwG5oaKjsDK2yB0iDHsRTvsK9zHvqJ9/HXsX60bmxBORhKDaJfaOguLNx/hha/3o4AnZgxlzrg+nV9LAKipgOVzjK6pN74P7u0brntV2ipMLiau6HuFlQMUwrlIYhBttjktn8fWJJOWW85Fg0N58lfDCA/0tk8wWsNn90FOEtz0EQS37xZQRW0Fqw+uZlrkNAI8W57AR4iuTBKDaLWs4kr+/kUqX+zJJiLIi7dvjucSW45z1Bo//xv2rICLH4GB7X/u4PNDn1NWU8ZN0TdZMTghnJMkBtGiqto6Fv1wmFc3pGHRmgcvHcT8Kf0677mEc9mzEtY9DNFXw6Q/tPs0Wmv+u/e/RAdFExsqM7UJIYlBnJO5zsJH2zN48ev95JRW84uYHvzfVTFEBNnptlFDB76GT++EPhPg1/9p9RwLTdl6fCtpxWk8MeEJeXZBCCQxiCZYLJovk47z/Ff7OJR/gtGRAbw8cxTj+gXbOzRD+lZYMRe6R8Os/4KbV4dO927SuwR6BHJ538utFKAQzk0SgzhFa833B/J5bt0+9mSWMKhHN/5zczyXRHd3nG/SOSmw7Hrw7QlzPgFP/w6dbnfebjZnbeb+0ffjaerkh/GEcFCSGAR1Fs265OP8+9s0kjJLCQvw4vnrR/KrUWG42qP76blkbof3rzNqCHNXQbeOz6z2+u7XCfAIYNaQWVYIUIiuQRLDeazGbGHVzkze+O4gh/JP0DfEh6evHc6vRoXhYbJzw/KZDn0Hy2eDdzDcvAoC+3T4lIl5iWzONGoL3m4O0G4ihIOQxHAeKjpRw4qEdBb/eITskipievnx6uxRXD6sl2PVEE5K/RxW3grBA4zbR34dH5RPa81zCc8R5BkktQUhziCJ4TyyJ6OEpT8dYc3uLKrNFsb1DeKfvx7OhYNCHacNoSGtjak5v/wzhMXB7BXgHWSVU689vJaduTv524S/SW1BiDNIYujiqmrrWLsnm6U/HWVXejHe7q5cGxfOzeP7MKSnn73DO7faSvj8Adj9Xxh0GVy7CDysMwZTRW0F/9r+L2KCY/jVgF9Z5ZxCdCWSGLogrTUJR4v4eHsGXyRmU1Ztpl+ID49dHcO1ceH4ebrZO8TmFR2BD+fC8T0w9SGY8qcOPadwpld2vkJuRS7PX/g8LsrG804L4YQkMXQhxwoq+GRnBp/syORYYQXe7q5cPqwX144O44J+wfYZ4K6tklfB5/cbA+LN/hAG/dKqp/85+2feT32f2UNmE9tdnnIWoimSGJzcobxyvkw6zpdJ2SRllqIUjO8XzH3TBnLZsJ74eDjJX3F5Hqz9I6Ssgt6jjFtH7RwQ71xKa0r5v83/R5RfFPfH3W/VcwvRlTjJp4Y4SWtNanYZX6Uc58s9x9mXUwZAbEQAf718CFeN7E1YQMeeBO5UWkPyp0ZSqC6DaY/ChPvA1bq/mnWWOv78/Z/Jr8hn6eVL8TI50Z+REJ1MEoMTKK2qZfOBfDbuy+W7/XnklFajFIzpE8SjV8Vw2bCe9HamZHBS1k5Y9wgc/cGoJcz4N/SIscmlXtzxIpszN/PY+McYHjrcJtcQoquQxOCAqs11JGaU8PPBAjal5bPjaBFmi8bX08TkgSFMHdSdqUNC6e7rpEM4lGTAN09C4nLjgbUrnoO4W61eSzhpWeoyFicvZubgmVw36DqbXEOIrkQSgwOoqjUSwZZDBfx8uIDtR4uoqrWgFET39GP+lH5MHdydUZEBuLk6cS+a/DT48SXYvRxQMOkBY+ngeEfNWb53OQu3LuSSyEv489g/2+w6QnQlkhg6mdaaowUV7EwvYuexYnYeKyY1uxSzRaMUDOnpx6yxkVzQL5hxfYMI8Ha3d8gdozVkbIOfXoWUNWDygFFzYdL9EBBpw8tq3kx8k9d2vcbU8Kk8M+UZ3FwcvJuuEA5CEoMN1ZgtpOWWk5JdSmp2KSlZpaQeL6W4ohYAH3dXRoQHMH9KP2IjAhjbFRLBSScKIPFD2LEU8lLBwx8mPwjj7rLK4HfNqait4Mmfn+TzQ59zdb+reXzC47i5SlIQorUkMViBuc7CscIKDuWd4GBeOftyykjNLiMtt4zaOg2Ap5sLg3v6cfmwnowID2BUZAADu/s65thE7VVZBPv+B6mfQdp6qKuBsHi4+iUYdi14+No8hOT8ZBZsWsDR0qP8Nva33DniTscc7kMIByaJoZXqLJrjpVWkF1Y0SgKH8so5VlhxKgEAhPp6ENPLjwsHhRLT24+YXn70DfHpWkkAjNtEefvg0EY48BUc/h4sZvALgzF3wKg50GNop4RSUl3CKztf4aP9HxHqFcqiXy5iTM8xnXJtIboaSQz1TlSbySmtIqe0mpzSKjKKKsgoqiS9qIL0wkqyiisxW05/+Lu5KvoE+9A/tBu/GNqTfiE+9O/ejf4h3fD37qK3LSx1kJsKGVvh2BY4/B2UZRv7ggfA+HshejqEjYZO+pZeXFXM+6nv80HqB5wwn+CGQTdw76h78fewXYO2EF3deZUYdqUXs+tYEcdLq8ktreJ4aRU5pVXkllZTVm0+q3xIN3fCA70ZEe7PlSN6ERHoTXigFxFB3kQEemFy5h5CLakuh7y9kJNsJIOcJOO5g5pyY793CPSdAv2mQv+LbNqQfKZaSy3bc7bz2cHPWHdkHdV11UyLnMbdI+9mcNDgTotDiK7K5olBKXUZ8BLgCryttV54xn4PYCkQBxQAN2qtj9gilrV7snnr+0O4uSq6+3rSw8+DQT18mTwwlB5+xnpPP0+6+3nQO8ALb/cunjerSqH4qDFoXdHR0+/z9hnvT3LzhtAhMHImhI+FiDEQ2LfTagUAeRV5bDu+jW/Tv+WHzB8oqy3Dx82HGf1nMHPITAYGDuy0WITo6pTWuuVS7T25Uq7AfuBSIAPYBszSWqc0KHMPMEJrfZdSaiZwjdb6xubOGx8frxMSEtocT+GJGrTWBHq7O8eAcm2hNdScgKqSBksxlOfCiVzjtTyn/rV+qSlrfA4PP2NmtOAB0H0odI82nkQOiLLq6KbNKaspI70snWNlxzhccpiUghRS8lPIrcwFIMgziAvDL2RqxFQu6HWBzKUgRBsopbZrreNbKmfrr8RjgTSt9aH6oJYDM4CUBmVmAI/Xv18JvKqUUtoGGSvIp51dQbU2Rvs89Xqu5Yz9FjNYaqHu5FJjbKurMdYbvq+rrS9bc7psbQXUVBhzE9RW1C+VDbZXGOMLnUwEuu7cP4NngNFNtFsP6B1rvHbrAYFRRjII6ANegW2qBWitqdN1mC3m04s2XmsttZgtZqrrqjlRe6LJpby2nKKqIvIr88mvzCevMo+S6pJT51coovyjGNtrLEODhzIydCRDQ4bKUNlC2JitE0MYkN5gPQMYd64yWmuzUqoECAbyrR3MO6vnsjp/BwC64f+fSkGNc5E+a70175v+YG1URjW9venzqvoP69OvWikwGYv2dgGCQIUY25UCXIxrKBdjOXUsaF0K5lIoToNi0Bm6wfUavNdNb0eDWZ/+4O8IL5MXQZ5BBHsFE+kbyejuownzDSPSN5II3wgifCOkRiCEHTjNTXSl1HxgPkBkZPsaOkMC+jGgIuvkGev7t6uT5z/7fYNtxrrxLfbkdlW/7exzuZwMGqVc6z+gFZx674Jycalfb1jm5L76dReT8f50xOf6s2myTKP3zdQEzlXuXNtNyoTJpfHi5uJmvD9jn4erBz5uPvi4+dDNrRvebt74uPngbfLGtcHPJoRwHLZODJlARIP18PptTZXJUEqZAH+MRuhGtNZvAW+B0cbQnmCmX/g3prfnQCGEOI/Y+mbtNmCgUqqvUsodmAmsOaPMGmBe/fvrgA22aF8QQgjROjatMdS3GdwLrMPorvqO1jpZKfUEkKC1XgMsAt5TSqUBhRjJQwghhJ3YvI1Ba70WWHvGtkcbvK8Crrd1HEIIIVpH+v0JIYRoRBKDEEKIRiQxCCGEaEQSgxBCiEYkMQghhGjEpoPo2YpSKg842mLBpoVgg+E2rMBR4wLHjU3iajtHjU3iapv2xtVHax3aUiGnTAwdoZRKaM3ogp3NUeMCx41N4mo7R41N4mobW8clt5KEEEI0IolBCCFEI+djYnjL3gGcg6PGBY4bm8TVdo4am8TVNjaN67xrYxBCCNG887HGIIQQohnnZWJQSsUqpX5WSu1SSiUopcbaO6aTlFK/U0rtVUolK6WesXc8Z1JK/UEppZVSIfaOBUAp9Wz9n1eiUupTpVSAneO5TCm1TymVppRaYM9YTlJKRSilNiqlUup/r+6zd0wNKaVclVI7lVKf2zuWhpRSAUqplfW/X6lKqfH2jglAKfVA/d9jklLqv0opT2tf47xMDMAzwN+01rHAo/XrdqeUughjDuyRWuuhwHN2DqkRpVQE8AvgmL1jaWA9MExrPQLYD/zVXoEopVyB14DLgRhgllIqxl7xNGAG/qC1jgEuAH7rIHGddB+Qau8gmvAS8D+t9RBgJA4Qo1IqDPg9EK+1HoYxnYHVpyo4XxODBvzq3/sDWc2U7Ux3Awu11tUAWutcO8dzpheAP3P2VNV2o7X+Smt9cvLpnzFmCbSXsUCa1vqQ1roGWI6R6O1Ka52ttd5R/74M4wMuzL5RGZRS4cCVwNv2jqUhpZQ/MAVjvhi01jVa62L7RnWKCfCqn/HSGxt8fp2vieF+4FmlVDrGt3K7fcs8wyBgslJqi1LqO6XUGHsHdJJSagaQqbXebe9YmnEb8KUdrx8GpDdYz8BBPoBPUkpFAaOALfaN5JQXMb5sWOwdyBn6AnnAu/W3ud5WSvnYOyitdSbGZ9YxIBso0Vp/Ze3r2HyiHntRSn0N9Gxi18PANOABrfXHSqkbML4VXOIAcZmAIIzq/hhghVKqX2dNddpCbA9h3EbqdM3FpbVeXV/mYYxbJss6MzZnopTqBnwM3K+1LnWAeK4CcrXW25VSU+0dzxlMwGjgd1rrLUqpl4AFwP/ZMyilVCBGLbQvUAx8pJSao7V+35rX6bKJQWt9zg96pdRSjPuaAB/RidXYFuK6G/ikPhFsVUpZMMZEybNnbEqp4Ri/iLuVUmDcrtmhlBqrtT5ur7gaxHcLcBUwzc7zhWcCEQ3Ww+u32Z1Syg0jKSzTWn9i73jqTQSmK6WuADwBP6XU+1rrOXaOC4zaXobW+mTNaiVGYrC3S4DDWus8AKXUJ8AEwKqJ4Xy9lZQFXFj//mLggB1jaWgVcBGAUmoQ4I4DDOCltd6jte6utY7SWkdh/KMZ3RlJoSVKqcswbkVM11pX2DmcbcBApVRfpZQ7RqPgGjvHhDKy+SIgVWv9L3vHc5LW+q9a6/D636mZwAYHSQrU/26nK6UG12+aBqTYMaSTjgEXKKW86/9ep2GDRvEuW2NowW+Al+obb6qA+XaO56R3gHeUUklADTDPzt+AncGrgAewvr4287PW+i57BKK1Niul7gXWYfQWeUdrnWyPWM4wEZgL7FFK7arf9lD9fOzi3H4HLKtP8oeAW+0cD/W3tVYCOzBune7EBk9By5PPQgghGjlfbyUJIYQ4B0kMQgghGpHEIIQQohFJDEIIIRqRxCCEEKIRSQxCCCEakcQghBCiEUkMQliBUmpM/ZwQnkopn6wDHi8AAADBSURBVPrx8ofZOy4h2kMecBPCSpRST2GM+eOFMc7O/7d3hzYIRTEUQFsPCoVkDSSDMAF7sAkDMAASw0iEFPExlfz85IVwzgTX3fS9pD0PjgSzKAZYyGd1wiOmNSv7qnoNjgSzeEqC5WwiYhUR65gmB/hJJgZYSGZeY7ratouIbVWdBkeCWf51uyosKjOPEfGsqsvn9vM9Mw9VdRudDb5lYgCg8ccAQKMYAGgUAwCNYgCgUQwANIoBgEYxANAoBgCaN1EAGU46hJgUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "w1 = 0.5\n",
    "w2 = 1.0\n",
    "w3 = 2.0\n",
    "l1 = 'w = 0.5'\n",
    "l2 = 'w = 1.0'\n",
    "l3 = 'w = 2.0'\n",
    "for w, l in [(w1, l1), (w2, l2), (w3, l3)]:\n",
    "    f = 1 / (1 + np.exp(-x*w))\n",
    "    plt.plot(x, f, label=l)\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('h_w(x)')\n",
    "plt.legend(loc=2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# when x>1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEKCAYAAAAW8vJGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xmc3HWZ4PHP0/eRdCfpztFJJ3QSAuSAcCSIgyAIGZKIYddBFkYURWX0ZWZ0VFxmXRhx3V0cxtkRZXYH5RxRdDyAGTEccoxCIIkhgRwQmqSTdJK+qquv6q6r69k/qjr0laSP+v2+XdXP+/XKq65f/56nO9311Pf8iapijDHG9MlxnYAxxpiJxQqDMcaYAawwGGOMGcAKgzHGmAGsMBhjjBnACoMxxpgBrDAYY4wZwAqDMcaYAawwGGOMGSDPdQJjUVlZqTU1Na7TMMaYjPLHP/6xRVVnnuq4jCwMNTU1bNu2zXUaxhiTUUTk4EiOs64kY4wxA1hhMMYYM4AVBmOMMQNk5BjDcGKxGPX19YTDYdepTHhFRUVUV1eTn5/vOhVjzASUNYWhvr6eqVOnUlNTg4i4TmfCUlUCgQD19fUsXLjQdTrGmAnI064kEXlARJpEZNcJXhcRuUdEakXkDRE5f6yxwuEwFRUVVhROQUSoqKiwlpUx5oS8HmN4CFh7ktfXAUtS/24B/u94gllRGBn7ORljTsbTriRV/Q8RqTnJIdcAj2jy+qKvisg0EalS1WNe5mXMeMQTcdoibbSF2whGgrRH2on0RoglYkR7o8QSMeKJOLFE7PjXqCqKHr8PoLz3HMp7r6Oc8JK7vTGI9UCsO3nbG4FEL2hv6jbx3u170Qfd1aGvDXl+MCXeq8QTSqw3QbxXSaiiqbwT2vc9nvxUOtwju7rwqHz4vFv4wHlXexrD9RjDPOBwv8f1qeeGFAYRuYVkq4IFCxb4ktxo1NXVcfXVV7Nr17C9Zmlx6NAhbrrpJtra2ujt7eWuu+5i/fr1Q47btGkTX/rSl+jt7eWzn/0st912m2c5TQZN3U384cgfePXoq7wdfJtDHYeIa9zzuEJfy67vDTSD3kFH0yi1BuyoLD68JesLw4ip6n3AfQCrVq3KoL+Q9Pn2t7/Nddddxxe+8AX27NnD+vXrqaurG3BMb28vX/ziF3n22Weprq5m9erVbNiwgWXLlrlJOoO91foW9795P88efJZe7WVm8UzOrjybDy34ELNKZjG9aDrTC6czrXAahbmF5OfmU5BTQH5OPvm5+eRKLiJy/A1ekONvgv2f6+vaO34/kYA3HoNXfgBNuwGBOStg9tlQsRimzIKSSiipgKJyyCuA3L5/+ZCTDzmpP+3j3YZy6vsiNHdG+OHv9/PTLYfoDMcpLcjlvAXTWTJ7CtXTS6goLWB6aQEzSgooKcylIDeH/Nwc8nOF/Lwc8nNy+p/uve/zvW899TMZmJ51b04srgvDEWB+v8fVqecyUjwe5+Mf/zjbt29n+fLlPPLII5SUlKTt/CJCR0cHAO3t7cydO3fIMVu2bOH0009n0aJFAFx//fU88cQTVhhGoTfRy4O7H+Te1++lKK+ITyz7BBsWb+D0aad7/wbWcQx+/Rdw4CWYtRw2fB/O/DCUVngbF3j+rUa+9q9v0N4TY92KOXz8faexqmY6+bm23GmycV0YngQ2ishjwPuA9nSML9z5b7vZc7Rj3Mn1t2xuGX/7keUnPebtt9/m/vvv5+KLL+bmm2/mn/7pn/ja17424Ji7776bRx99dMjXXnrppdxzzz0nPf83v/lN/vRP/5Tvf//7hEIhnnvuuSHHHDlyhPnz36u11dXVvPbaayc9r3lPb6KXr7z4FZ4//DxX1VzF7RfdTnlhuT/BW96BB9Ymxw8+cg+c/8l+n+y99aPf7+fbv9nLWXOm8rNbLmLJ7Km+xDUTk6eFQUR+ClwGVIpIPfC3QD6Aqv4/4ClgPVALdAOf9jIfr82fP5+LL74YgBtvvJF77rlnSGG49dZbufXWW8d0/p/+9Kd86lOf4qtf/SqbN2/mE5/4BLt27SInxz7Rpcvd2+7m+cPP87VVX+OTyz7pXxdHKACPfix5/3MvwKyz/IkLbNrVwP98ai9rl8/hH68/l6L8XN9im4nJ61lJN5zidQW+mO64p/pk75XBbyLDvamMpsXwjW98g9/85jcA7Nixg/vvv59NmzYB8P73v59wOExLSwuzZs06/jXz5s3j8OH3xvPr6+uZN2/e2L+pSeQX+37Bo3sf5calN3LT8pv8C5zohZ/dCJ3H4KZ/97Uo7D3WwZd/9jrnzp9mRcG8R1Uz7t8FF1ygg+3Zs2fIc346cOCAAvrKK6+oqupnPvMZ/fu///u0xli7dq0++OCDqpr8fquqqjSRSAw4JhaL6cKFC3X//v0aiUT0nHPO0V27dg05l+uf10TTHmnXP/nJn+jNm27WeG/c3+A7fqr6t2Wq23/sa9hEIqE33LdZz/vWM9rUEfY1tnED2KYjeI+1Pog0OvPMM7n33ntZunQpwWCQL3zhC2k9/3e/+11++MMfsnLlSm644QYeeughRISjR48en7aal5fHD37wA6666iqWLl3Kddddx/LlblpQmeTBXQ/SEe3g1tW3kpvj46fmeARe+J9QtRJWnrSBnXZ/qG3hlXcDbLz8dGZOLfQ1tpnYXA8+Z42amhreeustT2MsW7aMl19+ecjzc+fO5amnnjr+eP369cOubzDDa+pu4sd7fsz6hes5a4Z/3TgAbHsQ2g7B1f8IPo4VJRLKdza9RfX0Yj5+0cRbF2TcshaDmfQe3v0w8UScjedt9Ddwbxx+/11YeCks/pCvoV/c18SuIx389ZVnUJhn4wpmICsMZlJLaIJNBzZxSfUlzJ86/9RfkE4HXoJQE1x4i2/TUvs8seMo00vy2XDu0LUwxlhhMJPa9sbtNPU0sbbmZHs9emT3r6BgKpy+xtew4Vgvz+1pZO2KObZ4zQzLfivMpLapbhNFuUVcNv8yfwPHo7D33+Cs9ZBf5GvoF95qIhTt5epzrLVghmeFwUxa8UScZw8+y6XVl1KSn76tS0Zk/wsQbocVf+ZvXODf3zhG5ZQC3rdwhu+xTWawwmAmra0NW2kNt7J2oYNupF2/hKJpsOhyX8OGInF+91Yj61ZUkWfdSOYE7DcjTerq6lixYkXaz/vwww+zZMkSlixZwsMPPzzsMa2traxZs4YlS5awZs0agsFg2vPIRi8feZmCnAIumXeJv4FVofY5OGNtcmdUH712IEA4lmDtijm+xjWZxQrDBNba2sqdd97Ja6+9xpYtW7jzzjuHfdO/6667uOKKK3jnnXe44ooruOuuuxxkm3l2NO9geeVyivL87eOndT90B+C09/sbF9h+sI3cHOG8BdN8j20yhxWGNOrbdnvp0qVce+21dHd3j+t8Tz/9NGvWrGHGjBlMnz6dNWvWHN8rqb8nnniCm25K7u1z00038fjjj48r7mQQ7Y2yJ7CHlTNX+h/8cGq32+oLfQ/9x4NBllZNpaTA1raaE8vO347f3gYNb6b3nHPOhnUn/ySe7m23h9tC+8iRoZeraGxspKqqKpnmnDk0NjaO+NuarPa27iWWiDkqDFugsAxm+rvKOt6bYGd9G9deUO1rXJN5srMwOOL1ttsjISJ2NawR2Nm0E8BNYajfCvMu8HULDIC3GzvpjvZywWnTfY1rMk92FoZTfLL3Srq33Z43bx4vvvji8cf19fVcdtllQ7529uzZHDt2jKqqKo4dOzZgG24zvB3NO5hbOpeZJTP9DRzphKY9cKl3Hw5OZPuhNgDOX2CFwZycjTGk0aFDh9i8eTMAP/nJT/jABz4w5Jhbb72VHTt2DPk33NXbrrrqKp555hmCwSDBYJBnnnmGq666ashxGzZsOD5j6eGHH+aaa65J83eWfXY273TTWjjyR9CEk/GF1w8GqZxSSPX0Yt9jm8xihSGN0r3t9owZM7j99ttZvXo1q1ev5o477mDGjOSipM9+9rNs27YNgNtuu41nn32WJUuW8Nxzz3HbbbeN+3vJZg2hBpq6m1g5y8X4wtbkbfUFvofefijI+QumWVejOaXs7EpywKttt2+++WZuvvnmIc//6Ec/On6/oqKC3/3ud2mPna12NjseX6g8E4r97c5pDUWpC3Rz/YW2xbY5NWsxmEmntq0WQVgyfYn/wZv2Jme4+WxfYycAy6rKfI9tMo8VBjPp1LXXMXfKXApzfb5qWawH2g9Dpf8FaX9zCIBFM0t9j20yjxUGM+nUddRRU17jf+DAu4BCxem+h97f3EVRfg5zy23g2ZyaFQYzqSQ0wcGOgywsW+h/8MA7ydvKM3wPvb8lRE1FKTk5NvBsTs0Kg5lUmrqb6In3sLDcQWFoqU3eViz2PfT+5i4Wz5rie1yTmawwmEnlQPsBAGrKavwP3rIPyqqhwN9+/ki8l0Ot3SyutPEFMzJWGNLEq223v/71r7N8+XKWLl3KX/3VX6GqQ46xbbdHrq6jDsDRGMM7UOn/+MKhQDcJhUUzrcVgRsYKwwT2yiuv8PLLL/PGG2+wa9cutm7dyksvvTTkONt2e+QOtB+gNL+UmcU+b4WhmuxKcjC+8K7NSDKjZIUhjdK97baIEA6HiUajRCIRYrEYs2fPHnKcbbs9cnXtddSU1fi/+rerEaKdUOFgqmpLFwALrSvJjFBWrnz+zpbv8FZrelchnzXjLP7rhf/1pMeke9vt97///Vx++eVUVVWhqmzcuJGlS5cO+Vrbdnvk6jrqOH/2+f4HbtmXvHXQlfRuU4hZUwuZWpTve2yTmbKyMLiS7m23a2tr2bt3L/X19QCsWbOG3//+91xyyYkvRWnbbp9YT7yHY6FjjgaeU1NVHbUYFtv4ghmFrCwMp/pk75V0b7v961//mosuuogpU5J/1OvWrWPz5s1DCoNtuz0yhzoOAbiZqhqohfwSKJvna1hVZX9ziKvPqfI1rslsno8xiMhaEXlbRGpFZMi2nyKyQEReEJHXReQNEVnvdU5eSfe22wsWLOCll14iHo8Ti8V46aWXhu1Ksm23R+ZQZ7IwnFZ2mv/BW/fDjEW+X5ynvSdGe0/MxhfMqHj6WyoiucC9wDpgGXCDiCwbdNh/B36uqucB1wP/5GVOXkr3ttvXXnstixcv5uyzz2blypWsXLmSj3zkI4Btuz0WDaEGAKpKHXx6bj8C5f5fUvNoWxiAudNsKwwzcl53JV0I1KrqfgAReQy4BtjT7xgF+rZ8LAeOepyTJ7zYdjs3N5d//ud/HvY123Z79BpCDRTlFlFW4GCH0Y4jMH+172GPtfcAMKe8yPfYJnN5XRjmAYf7Pa4H3jfomG8Cz4jIXwKlwJUe52QmqcbuRuaUzvF/cD7aDT2tvo8vABxrT7UYbPM8MwoTYR3DDcBDqloNrAf+RUSG5CUit4jINhHZ1tzc7HuSJvM1hBqYXTp0HYjnOlKNYAddScfae8jNEWZO9XmLcZPRvC4MR4D5/R5Xp57r7zPAzwFUdTNQBFQOPpGq3qeqq1R11cyZw69aHW67CDPUZP05NYQamF3iojCkfuXL5voe+lhbmNlTC8m1XVXNKHhdGLYCS0RkoYgUkBxcfnLQMYeAKwBEZCnJwjDqJkFRURGBQGDSvumNlKoSCAQoKppcfc7xRJyWnhbmlM7xP/jxwuCmK6nKBp7NKHk6xqCqcRHZCDwN5AIPqOpuEfkWsE1VnwS+CvxQRP6a5ED0p3QM7+7V1dXU19dj3UynVlRURHW1/90aLrX0tNCrvZOvxdDew4p55b7HNZnN8wVuqvoU8NSg5+7od38PcPF44+Tn57NwoYOFSyYj9E1VddJiaD8CJRWQ7+8nd1XlWHuYNcscFEOT0SbC4LMxnmvsTu4f5awryUE3UrA7RiSeoMpmJJlRssJgJoW+FoObrqSjTgrD0bbkGoa50ybXeJIZPysMZlJoCDVQnFfsZnFbez2Uu1vDYC0GM1pWGMyk4G5xWwjCbU5aDA2pVc9VturZjJIVBjMpNIYa3XUjgZuupPYw+blC5RRb3GZGxwqDmRQaQg2OZiQlr6XhpCuprYfZZUXk2OI2M0pWGEzWiyViNPc0O24xuFjDELZuJDMmVhhM1mvpbkHRybnq2QaezRhYYTBZr28Nw6wSB1e26ziaXNyW528/v6rS0BG27bbNmFhhMFkv0BMAoLJ4yN6M3utqgin+d2F1hONE4wlm2sCzGQMrDCbrBcLJwlBRVOF/8FAzlA6/G7CXAl0RACqnFvge22Q+Kwwm6/UVhhnFM/wPHmqCKf53YbV0RQFsqqoZEysMJusFegKUF5aTn5Pvf/CuZih1URhSLQYrDGYMrDCYrNcabnXTjRQNQSwEU/zvSuorDBVTrCvJjJ4VBpP1Aj0BKoodFIaupuStixZDZwQRmFFihcGMnhUGk/Vaw63MKHIxvpC6aJSDweeWUJQZJQXk5dqfuBk9+60xWS/QE3DTldTXYnDRldQZsfEFM2ZWGExWi/RG6Ix1uulKCjnsSuqK2FRVM2ZWGExWa+1pBVytYWhJ3rroSuqKUlFqLQYzNlYYTFY7vrjN1eBz0TTI8/+Te0uXdSWZsbPCYLJa33YYbloMbha3dUfjdEd7rSvJjJkVBpPVWsPJriQnq567XG2HYauezfhYYTBZze0+SU1OCkNzanGbbaBnxsoKg8lqgZ4ApfmlFOU52H66q9nNPkmdturZjI8VBpPVnK1hiIUh0u5oqqp1JZnxscJgslog7Gg7jO7UVFXbJ8lkICsMJqs520DP4T5Jga4IZUV5FObl+h7bZAcrDCarBXoCk2+fpK4olVOtG8mMnRUGk7XiiThtkTa3O6s66Epq7opQaauezThYYTBZKxgOoqi7qapg+ySZjGSFwWQtp4vbQgHIL4WCEt9Dt4ZsnyQzPp4XBhFZKyJvi0itiNx2gmOuE5E9IrJbRH7idU5mcmiLtAEwrXCa/8G7A1Dif0sl3pugvSfG9FJrMZixy/Py5CKSC9wLrAHqga0i8qSq7ul3zBLgb4CLVTUoIv63vU1WCkaCAEwvnO5/8O4AlPjfUmnviaEKM0ocXN/aZA2vWwwXArWqul9Vo8BjwDWDjvkccK+qBgFUtcnjnMwk0RZOtRiKJk+LIdidXNxmLQYzHl4XhnnA4X6P61PP9XcGcIaIvCwir4rI2uFOJCK3iMg2EdnW3NzsUbommwTDyRaDm66kFiit9D1s3wZ6NsZgxmMiDD7nAUuAy4AbgB+KyJC/ZFW9T1VXqeqqmTP9nwJoMk8wEqSsoIy8HE97TIfX3eq4xWBdSWbsvC4MR4D5/R5Xp57rrx54UlVjqnoA2EeyUBgzLm3hNqYXORhfiIUh2uVkjKE1FANghnUlmXEY8UcpEVkFXALMBXqAXcCzfWMDJ7AVWCIiC0kWhOuBPx90zOMkWwoPikglya6l/SP+Dow5gWAk6KYbKXU5UacthhIrDGbsTtliEJFPi8h2kjOHioG3gSbgA8BzIvKwiCwY7mtVNQ5sBJ4G9gI/V9XdIvItEdmQOuxpICAie4AXgFtVNTDeb8yYYDjopsXQd63nEjdjDKUFuRTl2z5JZuxG0mIoITmVtGe4F0XkXJJdP4eGe11VnwKeGvTcHf3uK/CV1D9j0iYYCbKsYpn/gbtTn2sctRhsRpIZr1MWBlW990SviUiBqu5Ib0rGjJ+q0hZuczdVFZwUhtZQ1MYXzLiNePBZRF4UkZp+jy8kOYZgzITTE+8hmog6WtzmdozBxhfMeI1mHt//BjaJyD0k1yKsAz7tSVbGjFPfPklOxhi6WwCBYv9jB7qinD5ziu9xTXYZcWFQ1adF5PPAs0ALcJ6qNniWmTHj0LdPkrPtMIqnQa7/6ydsjMGkw2i6km4Hvg9cCnwTeFFEPuxRXsaMy/FVz5NoO4xwrJfuaK+NMZhxG81HmgrgwtTspM0isgn4EfAbTzIzZhyctxhsDYPJYCNuMajql/tPWVXVg6q6xpu0jBmfvhaDmzGGVidrGFpDycJgLQYzXiNZ4PZDETn7BK+VisjNIvLx9KdmzNgFI0HyJI8p+Q4GYkMtjrbDsMJg0mMkXUn3ArenisMuoBkoIrmorQx4AHjUswyNGYNgOMi0ommIiL+BVZ11Jb1XGGwDPTM+I1ngtgO4TkSmAKuAKpJ7Je1V1bc9zs+YMWmLtLnZJynSCYmYmzGGkI0xmPQYzXTVLhF5BTgLUOCAZ1kZM07BcJAZRQ6u9dy36tnBtRhau2OIwDQrDGacRjNd9cPAu8A9wA+AWhFZ51VixoyHs51VnW6HEWFacT65OT53n5msM5rpqt8FLlfVWgARWUxyqupvvUjMmPFwdi0GlxvohWK2uM2kxWgu1NPZVxRS9gOdac7HmHHrTfTSHm133GJwMytphnUjmTQ4ZYtBRD6aurtNRJ4Cfk5yjOFj2CZ6ZgLqjHaS0ITjFoP/YwzB7igLZpT4Htdkn5F0JX2k3/1G4IOp+33TVo2ZUFojqQ30XKx6DrVATj4UTvU9dCAU5dz5DlpJJuuMZLrqpwFEpEhVw96nZMz4tIWT22E43SfJ5/UTqkowZBvomfQYzeDzLhFpBH6f+vcHVW33Ji1jxi4YSW2H4epaDA4GnjsjceIJtTEGkxaj2SvpdOAG4E3gw8BOEbGrt5kJp6/F4GyModTd4jbbDsOkw2jWMVQDFwOXAOcBu4GfeZSXMWPW12JwMyupxUmLIWCFwaTRaLqSDpGchfS/VPXzHuVjzLgFw0GK84opynMwN8LVltt922FYYTBpMJp1DOcBjwB/LiKbReQREfmMR3kZM2ZtkTY34wu9cehpc7uBno0xmDQYzV5JO0XkXZLbYlwC3Ehy6ur9HuVmzJgEw0E34wvhNkCdrWEAmDHFCoMZvxEXBhHZBhQCr5CclXSpqh70KjFjxqpvy23fhVqStw5WPQdCUQpycygtyPU9tsk+oxljWKeqzSd6UURuUtWH05CTMeMSjASpKa/xP7DTfZKiTC/N9//6EyYrjWa66gmLQsqXxpmLMWnh7FoMTndWjdl1GEzajGbw+VTso4pxLtobJRQLTb6dVbujNlXVpE06C4Om8VzGjEkwnFr17KQw9I0xuJmVZIXBpIu1GExWaYukVj272g6jYArk+79+wgqDSafRzEoqBP4MqOn/dar6rdTdl9OamTFj4HbVc8DJjKR4b4L2HhtjMOkzmhbDE8A1QBwI9fsHgKpuHO6LRGStiLwtIrUictuJTi4ifyYiKiKrRpGTMQM43yfJQTdSW08MsO0wTPqMZrpqtaquHc3JRSQXuBdYA9QDW0XkSVXdM+i4qSRnNb02mvMbM1hrOHUtBheFIdQCpTN9D9tq+ySZNBtNi+EVETl7lOe/EKhV1f2qGgUeI9nqGOx/AN8B7HoPZlzaIm0IQllBmf/BHW25bYXBpNtILu35JskZR3nAp0VkPxAhOdisqnrOSb58HnC43+N64H2Dzn8+MF9VfyMit44yf2MGCIaDlBWWkZczmsZwmrjeQM/GGEyajOSv52qvgotIDvAPwKdGcOwtwC0ACxYs8Colk+GcbaAX64FYyMngc2u3tRhMeo3k0p7j2Q/pCDC/3+Pq1HN9pgIrgBdTS/nnAE+KyAZV3TYoj/uA+wBWrVplaybMsJxtoNe3uK3U/w30Wrv6ttzO9z22yU7pXMcwnK3AEhFZKCIFwPXAk30vqmq7qlaqao2q1gCvAkOKgjEjFYwEJ992GN1RphTmUZhnG+iZ9PC0MKhqHNgIPA3sBX6uqrtF5FsissHL2GZyagu3Tb7tMFIb6BmTLp6P0KnqU8BTg5674wTHXuZ1PiZ7qSqtkVY3LYaQu8IQCEXtAj0mrbzuSjLGNx3RDuKJOBVF/r85E2pK3jpYx9DSFaVySqHvcU32ssJgskYgnPzUXlHsoDB0NUFOPhT7343V0hWxwmDSygqDyRqBHoeFIdScbC34fKGcREJpDUWpnGpdSSZ9rDCYrHG8xeCiK6mrCab4340U7I7Sm1BrMZi0ssJgskZrT3KfJHcthlm+hw2kVj1bYTDpZIXBZI1AOECO5FBeUO5/8L6uJJ+1dEYAKwwmvawwmKwR6AkwvXA6uTk+L/RSTRYGB11JzV19hcHGGEz6WGEwWSMQDrjpRgq3QW/USVdSS5d1JZn0s8JgskZrT6ujgefm5O0UF4UhQl6OUF5sK59N+lhhMFnDWYvB5eK2zggVUwrIybFLrpv0scJgskZruJUZRf5ve00o1WJwUBgCIVv1bNLPCoPJCt2xbnriPY5WPbvtSrLCYNLNCoPJCsdXPbvaJ0lynGyg19JphcGknxUGkxWc75NUUgE+T5NV1eQGerYdhkkzKwwmK7htMbhZ9dwRjhPtTVBZai0Gk15WGExWcN5icLC4raVvcZu1GEyaWWEwWaGvMDi5epuj7TACtrjNeMQKg8kKgZ4A5YXl5Oc4WOjlqCvpeIvBCoNJMysMJiu0hh2teo50QazbbVeSFQaTZlYYTFYI9Lhe9eygxdAZIUdgRqmNMZj0ssJgskIgHHCz6rnL3arn5q4o00sKyLXtMEyaWWEwGU9VaepuYlaJ/5/a6TyavC2r8j10Y0eY2WVFvsc12c8Kg8l4HdEOeuI9zCmZ43/w9iPJ27J5voc+2tbD3GlWGEz6WWEwGa8h1ADAnFIHhaHjCOSXQLH/02SPtYepKi/2Pa7JflYYTMZr7G4EYHbpbP+DdxyBsrkg/vbzd0fjtPfEmFNuLQaTflYYTMY73mJw1ZXkoBvpWHsYwLqSjCesMJiM1xBqIFdyqSyu9D94x1Eor/Y9bEOqMMwps64kk35WGEzGawg1MLNkJrk+725Kbxy6GpJdST472tYDWIvBeMMKg8l4jd2NbrqROo+BJpx2Jdl0VeMFKwwm4zWEGhzNSEqtYXDQlXSsPUxFaQFF+T63ksykYIXBZDRVpbG7kdklLmYk1SdvHXQlHWvvocq6kYxHPC8MIrJWRN4WkVoRuW2Y178iIntE5A0R+Z2InOZ1TiZ7tEXaiPRG3LQYHC5uO9ZmaxiMdzwtDCKSC9wLrAOWATeIyLJBh70OrFJjcugoAAAPMklEQVTVc4BfAH/nZU4mu7hd3HYUCqZAUbnvoY+191BlaxiMR7xuMVwI1KrqflWNAo8B1/Q/QFVfUNXu1MNXAf87bE3G6isMzrqSHCxuC0XidITj1mIwnvG6MMwDDvd7XJ967kQ+A/x2uBdE5BYR2SYi25qbm9OYoslkfauenXUlOZmRlJyqai0G45UJM/gsIjcCq4C7h3tdVe9T1VWqumrmTP+3ODYTU0OogbycPDfXYug4CuXupqpaYTBeyfP4/EeA+f0eV6eeG0BErgS+AXxQVSMe52SySEN3A7NLZpMjPn/GiUehqxHKHExVbevbDsO6kow3vP5r2gosEZGFIlIAXA882f8AETkP+Gdgg6o2eZyPyTINoQY34wudxwB1s+o51ZU0q8wu6Wm84WlhUNU4sBF4GtgL/FxVd4vIt0RkQ+qwu4EpwL+KyA4RefIEpzNmiMMdh6me6mC+Quv+5O2Mhb6HPhjoZm55EYV5trjNeMPrriRU9SngqUHP3dHv/pVe52CyUygWoqmniYXl/r85E6hN3lYs8T30/uYuFs2c4ntcM3lMmMFnY0arrqMOgJqyGv+Dt7yTXMMw1d/ZUKrKu80hFs0s9TWumVysMJiMdaD9AOCqMOyDitN9X8PQ3BmhKxJnUaUVBuMdKwwmY9W115EjOSwoW+B/8EAtVJ7he9h3m0MALJ5lXUnGO1YYTMaq66hjbulcCnIL/A0c7Yb2w1DpYHyhpQvAxhiMp6wwmIxV115HTXmN/4Fb303eVpzue+h3m0IU5edQZddhMB6ywmAyUkITHOw46GZGUss7yVtHLYaFlVPIyfF3bMNMLlYYTEZqDDUS7g27GXjum6o6Y7HvoffbjCTjAysMJiP1zUhy02LYB+ULoKDE17DhWC/1wW4W2/iC8ZgVBpORDnS4nKr6DlT6P75wMNBNQmGxtRiMx6wwmIxU115HaX4plcWV/gZWTXYlOVrxDLCo0loMxltWGExGqm2rZVH5IsTnBWa0H4ZoF8z0fw3D242diGBjDMZzVhhMxoklYuwO7Oacmef4H/zwluRt9WrfQ28/1MaZs6dSWuj5FmdmkrPCYDLOvuA+euI9rJy50v/gh7dAfinMWu5r2ERCef1QkPMWTPc1rpmcrDCYjLOzaSeAm8JQvwXmnQ+5/n5qr23uojMc5/wF03yNayYnKwwm4+xs3sms4llUlVb5GzjaDQ1vuulGOhgE4ILTrMVgvGeFwWScnc07WTlrpf8Dz0dfh0Qc5l/ob1xg+6Eg00vyWWi7qhofWGEwGaWlp4UjXUfcdSOBs4Hn8xZM978YmknJCoPJKDubHY4vHN4KMxZBqb9rJ9q7Y9Q2ddn4gvGNFQaTUXY27SQ/J5+lFUv9DZxIJFsM89/nb1xg++Hk+ML5Nr5gfGKFwWQMVeX5w89z/uzzKcwt9Dd4/VYINcOiy/yNCzyzu5GSglzOm2+FwfjDCoPJGG+1vsXBjoOsq1nnf/Ddv4LcQjhzva9hY70JNu06xpVLZ1NckOtrbDN5WWEwGeO3db8lT/K48rQr/Q2c6IXdj8OSNVBU5mvol2tbCHbHuPocn6fmmknNCoPJCKrKM3XPcNHciygvLPc3+KHN0NUAKz7qb1zg3984xtTCPD545kzfY5vJywqDyQhvtrzJka4jrK1Z63/wXb+C/BI4w9/YkXgvT+9uYM3y2RTmWTeS8Y8VBpMRfvnOLynIKeBDCz7kb+Bwe3J84Yy1UODv4rLfvtlAZzjOR1bO9TWuMVYYzIS3v30/j9c+znVnXsfUgqn+Bn/l+9AThIu/5GvYaDzBPzy7j2VVZXxwiXUjGX9ZYTAT3ve3f5/ivGI+d87n/A3c2Qib74XlH4W55/oa+rGthzjU2s3X155JTo6tdjb+ssJgJrTXm17nuUPPcdPym5hRNMPf4C/+L+iNwof+u69h23ti3PO7d7ho0Qw+eIa1Foz/rDCYCash1MDXXvwaVaVVfHLZJ/0N/vqP4Y8PwYV/ARWLfQsb603wxUe309Yd42/WLbW9kYwTdikoMyF1RjvZ+LuNhOIhHln3CKX5Pg78vvsC/NuXYNHlsOZO38ImEsrtj+/iD7Ut3H3tOaycb3sjGTc8bzGIyFoReVtEakXktmFeLxSRn6Vef01EarzOyUxsO5t38rF/+xi1bbV894Pf5YzpPl1fOZGA3/8DPHotVJwOH3sIcvN9Cd3UEeaTD2zhsa2H+eLli/nYqvm+xDVmOJ62GEQkF7gXWAPUA1tF5ElV3dPvsM8AQVU9XUSuB74D/Bcv8zITj6qyq2UXP977Y56ue5rZJbN5aO1DnDvLh0HfWE9yZfPme6HxTVj2n+Aj/wjF3n9iP9LWw79sPshPXjtItDfBXR89m/+y2oqCccvrrqQLgVpV3Q8gIo8B1wD9C8M1wDdT938B/EBERFXV49yMz2KJGN2xbkKxEMFwkMbuRuo66tgX3MeWY1to7mmmNL+UP1/653x+5ecpK0jT9hOqyTf/aAgiHdDZAO310PI2HNkOB1+B3gjMPAs++kM4+2OQpr793oQSisbpjvQS7I7S1BnhYCDEvsZOthxoZV9jFzkCa1fM4StrzuT0WVPSEteY8fC6MMwDDvd7XA8M3rf4+DGqGheRdqACaEl3Mt98+Hq2RHcBoCf5u9cT3D/ZcSc7dshxqdhyilg64FkZ5rXhjhx41gHnH/BmN9wZhs9j4HEysuP6hVIgDkRPMO2yMp7gnHCM1T0xLg+1Urrve+hvvkf7oONkUMTBjwdG7DsGigmTO8yxMXI5wDxe40r+gwvY0rwCfi3w62c50eeSYZ89QRrxhNIT6x32talFeaysnsafnV/N+rOrmD+j5ATfizH+y5jBZxG5BbgFYMGCBWM6x7TiWcwJv/eJ7IS1QWTga4P+8OUkXz342ePvxYMq0bCP5ORnlZO8E/efvTK44Mgw95LnGxTthPGh7yei/e4PPePwXwOQi1CkuRSRQ6HmMkXzmZbIZ2aikNK+X8NC4VC/3bRVhv6Uhv4Ihslg0NfFcoqJ5JYQzSkmkltKV14FHQUzCRbMozcnOYYwT4T/fJLv5SSnT2Ux9MncHCgtzKO0II/SwjzKi/OZXVZI9fQSZpcV2owjM2F5XRiOAP07TKtTzw13TL2I5AHlQGDwiVT1PuA+gFWrVo2pm+nL190zli8zxphJxetZSVuBJSKyUEQKgOuBJwcd8yRwU+r+tcDzNr5gjDHueNpiSI0ZbASeBnKBB1R1t4h8C9imqk8C9wP/IiK1QCvJ4mGMMcYRz8cYVPUp4KlBz93R734Y+JjXeRhjjBkZ2xLDGGPMAFYYjDHGDGCFwRhjzABWGIwxxgxghcEYY8wAkolLBkSkGTg4xi+vxIPtNtJgouYFEzc3y2v0JmpultfojDWv01T1lFd/ysjCMB4isk1VV7nOY7CJmhdM3Nwsr9GbqLlZXqPjdV7WlWSMMWYAKwzGGGMGmIyF4T7XCZzARM0LJm5ultfoTdTcLK/R8TSvSTfGYIwx5uQmY4vBGGPMSUzKwiAi54rIqyKyQ0S2iciFrnPqIyJ/KSJvichuEfk71/kMJiJfFREVkUrXuQCIyN2pn9cbIvJrEfH+Qs0nz2etiLwtIrUicpvLXPqIyHwReUFE9qR+r77kOqf+RCRXRF4XkX93nUt/IjJNRH6R+v3aKyLvd50TgIj8der/cZeI/FREitIdY1IWBuDvgDtV9VzgjtRj50TkcpLXwF6pqsuBv3ec0gAiMh/4U+CQ61z6eRZYoarnAPuAv3GViIjkAvcC64BlwA0issxVPv3Ega+q6jLgIuCLEySvPl8C9rpOYhjfAzap6lnASiZAjiIyD/grYJWqriB5OYO0X6pgshYGBfquNF8OHHWYS39fAO5S1QiAqjY5zmew/wN8nZNf7tlXqvqMqsZTD18leZVAVy4EalV1v6pGgcdIFnqnVPWYqm5P3e8k+QY3z21WSSJSDXwY+JHrXPoTkXLgUpLXi0FVo6ra5jar4/KA4tQVL0vw4P1rshaGLwN3i8hhkp/KnX3KHOQM4BIReU1EXhKR1a4T6iMi1wBHVHWn61xO4mbgtw7jzwMO93tczwR5A+4jIjXAecBrbjM57h9JfthIuE5kkIVAM/BgqpvrRyJS6jopVT1C8j3rEHAMaFfVZ9Idx/ML9bgiIs8Bc4Z56RvAFcBfq+ovReQ6kp8KrpwAeeUBM0g291cDPxeRRX5d6vQUuf03kt1IvjtZXqr6ROqYb5DsMnnUz9wyiYhMAX4JfFlVOyZAPlcDTar6RxG5zHU+g+QB5wN/qaqvicj3gNuA210mJSLTSbZCFwJtwL+KyI2q+uN0xsnawqCqJ3yjF5FHSPZrAvwrPjZjT5HXF4BfpQrBFhFJkNwTpdllbiJyNslfxJ0iAsnumu0icqGqNrjKq19+nwKuBq5wfL3wI8D8fo+rU885JyL5JIvCo6r6K9f5pFwMbBCR9UARUCYiP1bVGx3nBcnWXr2q9rWsfkGyMLh2JXBAVZsBRORXwJ8AaS0Mk7Ur6SjwwdT9DwHvOMylv8eBywFE5AyggAmwgZeqvqmqs1S1RlVrSP7RnO9HUTgVEVlLsitig6p2O05nK7BERBaKSAHJQcEnHeeEJKv5/cBeVf0H1/n0UdW/UdXq1O/U9cDzE6QokPrdPiwiZ6aeugLY4zClPoeAi0SkJPX/egUeDIpnbYvhFD4HfC81eBMGbnGcT58HgAdEZBcQBW5y/Ak4E/wAKASeTbVmXlXVz7tIRFXjIrIReJrkbJEHVHW3i1wGuRj4BPCmiOxIPfffUtdjNyf2l8CjqSK/H/i043xIdWv9AthOsuv0dTxYBW0rn40xxgwwWbuSjDHGnIAVBmOMMQNYYTDGGDOAFQZjjDEDWGEwxhgzgBUGY4wxA1hhMMYYM4AVBmPSQERWp64JUSQipan98le4zsuYsbAFbsakiYh8m+SeP8Uk99n5345TMmZMrDAYkyaprRO2ktxm5U9UtddxSsaMiXUlGZM+FcAUYCrJloMxGclaDMakiYg8SfKqbQuBKlXd6DglY8Zksu6uakxaicgngZiq/iR17edXRORDqvq869yMGS1rMRhjjBnAxhiMMcYMYIXBGGPMAFYYjDHGDGCFwRhjzABWGIwxxgxghcEYY8wAVhiMMcYMYIXBGGPMAP8f3eP9RYA6fGkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "w = 5.0\n",
    "b1 = -8.0\n",
    "b2 = 0.0\n",
    "b3 = 8.0\n",
    "l1 = 'b = -8.0'\n",
    "l2 = 'b = 0.0'\n",
    "l3 = 'b = 8.0'\n",
    "for b, l in [(b1, l1), (b2, l2), (b3, l3)]:\n",
    "    f = 1 / (1 + np.exp(-(x*w+b)))\n",
    "    plt.plot(x, f, label=l)\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('h_wb(x)')\n",
    "plt.legend(loc=2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#feed forward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.2, 0.2, 0.2],\n",
       "       [0.4, 0.4, 0.4],\n",
       "       [0.6, 0.6, 0.6]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "w1 = np.array([[0.2, 0.2, 0.2], [0.4, 0.4, 0.4], [0.6, 0.6, 0.6]])\n",
    "w1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.5, 0.5, 0.5]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2 = np.zeros((1, 3))\n",
    "w2[0,:] = np.array([0.5, 0.5, 0.5])\n",
    "w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.8 0.8 0.8]\n",
      "[0.2]\n"
     ]
    }
   ],
   "source": [
    "b1 = np.array([0.8, 0.8, 0.8])\n",
    "b2 = np.array([0.2])\n",
    "print(b1)\n",
    "print(b2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x):\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_looped_nn_calc(n_layers, x, w, b):\n",
    "    for l in range(n_layers-1):\n",
    "        #Setup the input array which the weights will be multiplied by for each layer\n",
    "        #If it's the first layer, the input array will be the x input vector\n",
    "        #If it's not the first layer, the input to the next layer will be the \n",
    "        #output of the previous layer\n",
    "        if l == 0:\n",
    "            node_in = x\n",
    "        else:\n",
    "            node_in = h\n",
    "        #Setup the output array for the nodes in layer l + 1\n",
    "        h = np.zeros((w[l].shape[0],))\n",
    "        #loop through the rows of the weight array\n",
    "        for i in range(w[l].shape[0]):\n",
    "            #setup the sum inside the activation function\n",
    "            f_sum = 0\n",
    "            #loop through the columns of the weight array\n",
    "            for j in range(w[l].shape[1]):\n",
    "                f_sum += w[l][i][j] * node_in[j]\n",
    "            #add the bias\n",
    "            f_sum += b[l][i]\n",
    "            #finally use the activation function to calculate the\n",
    "            #i-th output i.e. h1, h2, h3\n",
    "            h[i] = f(f_sum)\n",
    "    return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = [w1, w2]\n",
    "b = [b1, b2]\n",
    "#a dummy x input vector\n",
    "x = [1.5, 2.0, 3.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[0.2, 0.2, 0.2],\n",
       "        [0.4, 0.4, 0.4],\n",
       "        [0.6, 0.6, 0.6]]), array([[0.5, 0.5, 0.5]])]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0.8, 0.8, 0.8]), array([0.2])]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1.5, 2.0, 3.0]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.83545566])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_looped_nn_calc(3, x, w, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 µs ± 907 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit simple_looped_nn_calc(3, x, w, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def matrix_feed_forward_calc(n_layers, x, w, b):\n",
    "    for l in range(n_layers-1):\n",
    "        if l == 0:\n",
    "            node_in = x\n",
    "        else:\n",
    "            node_in = h\n",
    "        z = w[l].dot(node_in) + b[l]\n",
    "        h = f(z)\n",
    "    return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#GD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The local minimum occurs at 2.249965\n",
      "8.986384145703852e-06\n"
     ]
    }
   ],
   "source": [
    "x_old = 0 # The value does not matter as long as abs(x_new - x_old) > precision\n",
    "x_new = 6 # The algorithm starts at x=6\n",
    "gamma = 0.01 # step size\n",
    "precision = 0.00001\n",
    "\n",
    "def df(x):\n",
    "    y = 4 * x**3 - 9 * x**2\n",
    "    return y\n",
    "\n",
    "while abs(x_new - x_old) > precision:\n",
    "    x_old = x_new\n",
    "    x_new += -gamma * df(x_old)\n",
    "\n",
    "print(\"The local minimum occurs at %f\" % x_new)\n",
    "print(x_new - x_old)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1797, 64)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAC4lJREFUeJzt3d+LXPUdxvHn6SZBq2FXqhUxYizUgAjdBAkVRfODSKyS3PQiAQWlJb1oxdCCaG+i/4DYiyKEqBGMEY2GFGmtAbOI0GqTuNaYjUVDxCzqKmYT9aJB/fRiTmQb0u7ZZb/fndnP+wVDZncn83w24ZlzzsyZ+ToiBCCX7832AADqo/hAQhQfSIjiAwlRfCAhig8k1BXFt73W9ru237N9f+Gsx22P2T5UMmdC3hW299k+bPsd2/cWzjvP9hu232ryHiqZ12T22X7T9ouls5q8Y7bftj1se3/hrAHbu2wfsT1i+/qCWUua3+nM5ZTtzUXCImJWL5L6JL0v6UeSFkh6S9I1BfNukrRM0qFKv99lkpY11xdK+lfh38+SLmyuz5f0uqSfFv4dfyvpaUkvVvo3PSbp4kpZT0r6ZXN9gaSBSrl9kj6WdGWJ+++GLf5ySe9FxNGIOC3pGUnrS4VFxKuSPi91/+fI+ygiDjbXv5A0IunygnkREV82X85vLsXO0rK9SNJtkraVypgttvvV2VA8JkkRcToixivFr5b0fkR8UOLOu6H4l0v6cMLXx1WwGLPJ9mJJS9XZCpfM6bM9LGlM0t6IKJn3iKT7JH1bMONsIell2wdsbyqYc5WkTyU90RzKbLN9QcG8iTZI2lnqzruh+CnYvlDS85I2R8SpklkR8U1EDEpaJGm57WtL5Ni+XdJYRBwocf//x40RsUzSrZJ+bfumQjnz1DksfDQilkr6SlLR56AkyfYCSeskPVcqoxuKPyrpiglfL2q+N2fYnq9O6XdExAu1cpvd0n2S1haKuEHSOtvH1DlEW2X7qUJZ34mI0ebPMUm71TlcLOG4pOMT9ph2qfNAUNqtkg5GxCelArqh+P+Q9GPbVzWPdBsk/WmWZ5oxtq3OMeJIRDxcIe8S2wPN9fMlrZF0pERWRDwQEYsiYrE6/2+vRMQdJbLOsH2B7YVnrku6RVKRV2gi4mNJH9pe0nxrtaTDJbLOslEFd/Olzq7MrIqIr23/RtJf1Xkm8/GIeKdUnu2dklZIutj2cUlbIuKxUnnqbBXvlPR2c9wtSb+PiD8XyrtM0pO2+9R5YH82Iqq8zFbJpZJ2dx5PNU/S0xHxUsG8eyTtaDZKRyXdXTDrzIPZGkm/KprTvHQAIJFu2NUHUBnFBxKi+EBCFB9IiOIDCXVV8QuffjlrWeSR1215XVV8STX/cav+R5JHXjfldVvxAVRQ5AQe25wVNIOuvvrqKf+dkydPqr+/f1p58+ZN/YTOEydO6KKLLppW3ujo1N+acfr0aS1YsGBaeSdPnpzW3+sVEeHJbkPxe8DQ0FDVvIGBgap5W7ZsqZq3Z8+eqnm1tSk+u/pAQhQfSIjiAwlRfCAhig8kRPGBhCg+kBDFBxJqVfyaS1wBKG/S4jcf2vhHdT7y9xpJG21fU3owAOW02eJXXeIKQHltip9miSsgixn7XP3mgwNqv2cZwDS0KX6rJa4iYqukrRLvzgO6XZtd/Tm9xBWQ0aRb/NpLXAEor9UxfrPOW6m13gBUxpl7QEIUH0iI4gMJUXwgIYoPJETxgYQoPpAQxQcSmrE36aCc8fHxqnk333xz1byVK1dWzZvrK+m0wRYfSIjiAwlRfCAhig8kRPGBhCg+kBDFBxKi+EBCFB9IiOIDCbVZQutx22O2D9UYCEB5bbb42yWtLTwHgIomLX5EvCrp8wqzAKiEY3wgIdbOAxKaseKzdh7QO9jVBxJq83LeTkl/k7TE9nHbvyg/FoCS2iyaubHGIADqYVcfSIjiAwlRfCAhig8kRPGBhCg+kBDFBxKi+EBCrJ03DYODg1XzVqxYUTWvtuHh4dkeIR22+EBCFB9IiOIDCVF8ICGKDyRE8YGEKD6QEMUHEqL4QEIUH0iozYdtXmF7n+3Dtt+xfW+NwQCU0+Zc/a8l/S4iDtpeKOmA7b0RcbjwbAAKabN23kcRcbC5/oWkEUmXlx4MQDlTOsa3vVjSUkmvlxgGQB2t35Zr+0JJz0vaHBGnzvFz1s4DekSr4tuer07pd0TEC+e6DWvnAb2jzbP6lvSYpJGIeLj8SABKa3OMf4OkOyWtsj3cXH5WeC4ABbVZO+81Sa4wC4BKOHMPSIjiAwlRfCAhig8kRPGBhCg+kBDFBxKi+EBCc2LtvM2bN1fNe/DBB6vm9ff3V82rbWhoaLZHSIctPpAQxQcSovhAQhQfSIjiAwlRfCAhig8kRPGBhCg+kBDFBxJq8ym759l+w/Zbzdp5D9UYDEA5bc7V/7ekVRHxZfP5+q/Z/ktE/L3wbAAKafMpuyHpy+bL+c2FBTOAHtbqGN92n+1hSWOS9kYEa+cBPaxV8SPim4gYlLRI0nLb1559G9ubbO+3vX+mhwQws6b0rH5EjEvaJ2ntOX62NSKui4jrZmo4AGW0eVb/EtsDzfXzJa2RdKT0YADKafOs/mWSnrTdp84DxbMR8WLZsQCU1OZZ/X9KWlphFgCVcOYekBDFBxKi+EBCFB9IiOIDCVF8ICGKDyRE8YGE3HnX7QzfqT2n37Y7MDBQNe/EiRNV82pburTu+WHDw8NV82qLCE92G7b4QEIUH0iI4gMJUXwgIYoPJETxgYQoPpAQxQcSovhAQhQfSKh18ZtFNd60zQdtAj1uKlv8eyWNlBoEQD1tl9BaJOk2SdvKjgOghrZb/Eck3Sfp24KzAKikzUo6t0sai4gDk9yOtfOAHtFmi3+DpHW2j0l6RtIq20+dfSPWzgN6x6TFj4gHImJRRCyWtEHSKxFxR/HJABTD6/hAQm0WzfxORAxJGioyCYBq2OIDCVF8ICGKDyRE8YGEKD6QEMUHEqL4QEIUH0hoSifwACUMDg5WzZvra+e1wRYfSIjiAwlRfCAhig8kRPGBhCg+kBDFBxKi+EBCFB9IiOIDCbU6Zbf5aO0vJH0j6Ws+QhvobVM5V39lRHxWbBIA1bCrDyTUtvgh6WXbB2xvKjkQgPLa7urfGBGjtn8oaa/tIxHx6sQbNA8IPCgAPaDVFj8iRps/xyTtlrT8HLdh7TygR7RZLfcC2wvPXJd0i6RDpQcDUE6bXf1LJe22feb2T0fES0WnAlDUpMWPiKOSflJhFgCV8HIekBDFBxKi+EBCFB9IiOIDCVF8ICGKDyRE8YGEKD6QEMUHEqL4QEIUH0iI4gMJUXwgIYoPJETxgYQoPpAQxQcSovhAQq2Kb3vA9i7bR2yP2L6+9GAAymm7oMYfJL0UET+3vUDS9wvOBKCwSYtvu1/STZLukqSIOC3pdNmxAJTUZlf/KkmfSnrC9pu2tzULa/wX25ts77e9f8anBDCj2hR/nqRlkh6NiKWSvpJ0/9k3YgktoHe0Kf5xSccj4vXm613qPBAA6FGTFj8iPpb0oe0lzbdWSzpcdCoARbV9Vv8eSTuaZ/SPSrq73EgASmtV/IgYlsSxOzBHcOYekBDFBxKi+EBCFB9IiOIDCVF8ICGKDyRE8YGE2p65hwnGx8er5u3Zs6dq3vr166vmrVixomre9u3bq+Z1I7b4QEIUH0iI4gMJUXwgIYoPJETxgYQoPpAQxQcSovhAQpMW3/YS28MTLqdsb64xHIAyJj1lNyLelTQoSbb7JI1K2l14LgAFTXVXf7Wk9yPigxLDAKhjqsXfIGlniUEA1NO6+M1n6q+T9Nz/+Dlr5wE9Yipvy71V0sGI+ORcP4yIrZK2SpLtmIHZABQylV39jWI3H5gTWhW/WRZ7jaQXyo4DoIa2S2h9JekHhWcBUAln7gEJUXwgIYoPJETxgYQoPpAQxQcSovhAQhQfSIjiAwk5YubfT2P7U0nTec/+xZI+m+FxuiGLPPJq5V0ZEZdMdqMixZ8u2/sj4rq5lkUeed2Wx64+kBDFBxLqtuJvnaNZ5JHXVXlddYwPoI5u2+IDqIDiAwlRfCAhig8kRPGBhP4Dc36A4u8EGZ0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "digits = load_digits()\n",
    "print(digits.data.shape)\n",
    "import matplotlib.pyplot as plt \n",
    "plt.gray() \n",
    "plt.matshow(digits.images[1]) \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        , -0.33501649, -0.04308102,  0.27407152, -0.66447751,\n",
       "       -0.84412939, -0.40972392, -0.12502292, -0.05907756, -0.62400926,\n",
       "        0.4829745 ,  0.75962245, -0.05842586,  1.12772113,  0.87958306,\n",
       "       -0.13043338, -0.04462507,  0.11144272,  0.89588044, -0.86066632,\n",
       "       -1.14964846,  0.51547187,  1.90596347, -0.11422184, -0.03337973,\n",
       "        0.48648928,  0.46988512, -1.49990136, -1.61406277,  0.07639777,\n",
       "        1.54181413, -0.04723238,  0.        ,  0.76465553,  0.05263019,\n",
       "       -1.44763006, -1.73666443,  0.04361588,  1.43955804,  0.        ,\n",
       "       -0.06134367,  0.8105536 ,  0.63011714, -1.12245711, -1.06623158,\n",
       "        0.66096475,  0.81845076, -0.08874162, -0.03543326,  0.74211893,\n",
       "        1.15065212, -0.86867056,  0.11012973,  0.53761116, -0.75743581,\n",
       "       -0.20978513, -0.02359646, -0.29908135,  0.08671869,  0.20829258,\n",
       "       -0.36677122, -1.14664746, -0.5056698 , -0.19600752])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "X_scale = StandardScaler()\n",
    "X = X_scale.fit_transform(digits.data)\n",
    "X[0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "y = digits.target\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, array([0., 0., 0., 1., 0., 0., 0., 0., 0., 0.]))"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "def convert_y_to_vect(y):\n",
    "    y_vect = np.zeros((len(y), 10))\n",
    "    for i in range(len(y)):\n",
    "        y_vect[i, y[i]] = 1\n",
    "    return y_vect\n",
    "y_v_train = convert_y_to_vect(y_train)\n",
    "y_v_test = convert_y_to_vect(y_test)\n",
    "y_train[0], y_v_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_structure = [8*8, 30, 10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "def f_deriv(x):\n",
    "    return f(x) * (1 - f(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "#initalise \n",
    "import numpy.random as r\n",
    "def setup_and_init_weights(nn_structure):\n",
    "    W = {}\n",
    "    b = {}\n",
    "    for l in range(1, len(nn_structure)):\n",
    "        W[l] = r.random_sample((nn_structure[l], nn_structure[l-1]))\n",
    "        b[l] = r.random_sample((nn_structure[l],))\n",
    "        #print(W.shape, b.shape)\n",
    "    \n",
    "    return W, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_tri_values(nn_structure):\n",
    "    tri_W = {}\n",
    "    tri_b = {}\n",
    "    for l in range(1, len(nn_structure)):\n",
    "        tri_W[l] = np.zeros((nn_structure[l], nn_structure[l-1]))\n",
    "        tri_b[l] = np.zeros((nn_structure[l],))\n",
    "    #print(tri_W, tri_b)\n",
    "    return tri_W, tri_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feed_forward(x, W, b):\n",
    "    h = {1: x}\n",
    "    z = {}\n",
    "    for l in range(1, len(W) + 1):\n",
    "        # if it is the first layer, then the input into the weights is x, otherwise, \n",
    "        # it is the output from the last layer\n",
    "        if l == 1:\n",
    "            node_in = x\n",
    "        else:\n",
    "            node_in = h[l]\n",
    "        z[l+1] = W[l].dot(node_in) + b[l] # z^(l+1) = W^(l)*h^(l) + b^(l)  \n",
    "        h[l+1] = f(z[l+1]) # h^(l) = f(z^(l)) \n",
    "    #print(z, h)\n",
    "    return h, z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_out_layer_delta(y, h_out, z_out):\n",
    "    # delta^(nl) = -(y_i - h_i^(nl)) * f'(z_i^(nl))\n",
    "    return -(y-h_out) * f_deriv(z_out)\n",
    "\n",
    "def calculate_hidden_delta(delta_plus_1, w_l, z_l):\n",
    "    # delta^(l) = (transpose(W^(l)) * delta^(l+1)) * f'(z^(l))\n",
    "    return np.dot(np.transpose(w_l), delta_plus_1) * f_deriv(z_l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_nn(nn_structure, X, y, iter_num=300, alpha=0.25):\n",
    "    W, b = setup_and_init_weights(nn_structure)\n",
    "    cnt = 0\n",
    "    m = len(y)\n",
    "    avg_cost_func = []\n",
    "    print('Starting gradient descent for {} iterations'.format(iter_num))\n",
    "    while cnt < iter_num:\n",
    "        if cnt%1000 == 0:\n",
    "            print('Iteration {} of {}'.format(cnt, iter_num))\n",
    "        tri_W, tri_b = init_tri_values(nn_structure)\n",
    "        avg_cost = 0\n",
    "        for i in range(len(y)):\n",
    "            delta = {}\n",
    "            # perform the feed forward pass and return the stored h and z values, to be used in the\n",
    "            # gradient descent step\n",
    "            h, z = feed_forward(X[i, :], W, b)\n",
    "            # loop from nl-1 to 1 backpropagating the errors\n",
    "            for l in range(len(nn_structure), 0, -1):\n",
    "                if l == len(nn_structure):\n",
    "                    delta[l] = calculate_out_layer_delta(y[i,:], h[l], z[l])\n",
    "                    avg_cost += np.linalg.norm((y[i,:]-h[l]))\n",
    "                else:\n",
    "                    if l > 1:\n",
    "                        delta[l] = calculate_hidden_delta(delta[l+1], W[l], z[l])\n",
    "                    # triW^(l) = triW^(l) + delta^(l+1) * transpose(h^(l))\n",
    "                    tri_W[l] += np.dot(delta[l+1][:,np.newaxis], np.transpose(h[l][:,np.newaxis])) \n",
    "                    # trib^(l) = trib^(l) + delta^(l+1)\n",
    "                    tri_b[l] += delta[l+1]\n",
    "        # perform the gradient descent step for the weights in each layer\n",
    "        for l in range(len(nn_structure) - 1, 0, -1):\n",
    "            W[l] += -alpha * (1.0/m * tri_W[l])\n",
    "            b[l] += -alpha * (1.0/m * tri_b[l])\n",
    "        # complete the average cost calculation\n",
    "        avg_cost = 1.0/m * avg_cost\n",
    "        avg_cost_func.append(avg_cost)\n",
    "        cnt += 1\n",
    "    return W, b, avg_cost_func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting gradient descent for 300 iterations\n",
      "Iteration 0 of 300\n"
     ]
    }
   ],
   "source": [
    "W, b, avg_cost_func = train_nn(nn_structure, X_train, y_v_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4HeWZ9/HvrWLLRe6SXCRZ7oViY8s2NsaFXhJ6M4SWgGNgk5DsbkJY9k3ed3ezJAQ2hZLQQuihmBrAQMC4YGzLveMuyU1y70XS/f5xxlrFUbeORkf6fa7rXDpnznPm3MPI/PTMM/OMuTsiIiIAcWEXICIiDYdCQURESikURESklEJBRERKKRRERKSUQkFEREopFEREpJRCQURESikURESkVELYBdRUp06dPCsrK+wyRERiyrx587a7e0pV7WIuFLKyssjJyQm7DBGRmGJmG6vTToePRESklEJBRERKKRRERKSUQkFEREopFEREpJRCQURESikURESkVJMJhQ3bD/Dwx6uYuWY7h48Vh12OiEiDFLWL18wsA3geSAMceNLdf3tCm7bAi0BmUMuv3f1P0ahnyaY9PD51Lb//bA3N4uMYnNmOkT07MrJXRwZntCMpMT4aXysiElPM3aOzYrMuQBd3n29mycA84Ap3X16mzf1AW3f/iZmlAKuAzu5+tKL1Zmdne22vaN53+Bg5G3Yxa90OZq3dwbLNeyhxaJ4Qx5DM9ozsFQmJQentaJbQZDpRItIEmNk8d8+uql3UegruvgXYEjzfZ2YrgG7A8rLNgGQzM6A1sBMoilZNyUmJjO+fyvj+qQDsOXSMOet3MmvtDmat28Ejn3wNn0BSYhwjenRkTN8UxvTpRO/U1kRKFBFp3KLWU/i7LzHLAqYBp7r73jLLk4F3gf5AMnC9u/+1nM9PBCYCZGZmDt24sVpTeNTYrgNHmb1+J7PWbmf66u2s234AgC5tkxjTJ4Wz+3ZidO9OtGvZLCrfLyISLdXtKUQ9FMysNfAF8F/uPvmE964BzgJ+BPQCPgEGlQ2OE53M4aOaytt5kOmrtzN9dSEz1mxn3+EizOD09HaM7dOJs/umcEZGOxLidahJRBq2BhEKZpYIvA9McfdHynn/r8CD7j49eP0ZcJ+7z6lonfUZCmUVFZewKH8P074uZPrqQhbm7abEoU1SAuP6pXLugFTG9U2lbcvEeq9NRKQqoY8pBOMEzwAryguEQC5wLjDdzNKAfsC6aNV0MhLi4xjavT1Du7fnh+f3Zc+hY8xcs53PVhbw+coC3l20mfg4Y2j39pzbP5VzB6TRK6WVxiJEJKZE8+yj0cB0YAlQEiy+n8jpp7j7H8ysK/Ac0AUwIr2GFytbb1g9hcqUlDiL8nfztxUF/G1lASu2RI5+de/YknP6p3LegDSGZXXQGU0iEpoGcfgoGhpiKJxo0+5DfLaygM9WbGPm2h0cLSohOSmB8wakceEpnRnbN4UWzXRdhIjUH4VCA3HwaBEz1+zg42Vb+WTFNnYfPEZSYhzj+qZy0amdGd8/lbYtNA4hItEV+piCRLRslsD5A9M4f2AaRcUlzFm/k4+WbWXKsq18tGwrifHGyF6duOiUzpw/MI2U5OZhlywiTZh6CiEpKXEW5u9mytKtfLh0K7k7D2IGw7I68M3Tu3DxaV3o1FoBISJ1Q4ePYoi7s3LrPj5cupUPlmxhTcF+4gxG9erEN07vwkWndtYFcyJyUhQKMcrdWbVtH+8v2sJ7izezccdBEuKMs/t04puDunL+wDSSkzQGISI1o1BoBNydpZv28t7izfx18RY27T5Es4Q4xvVNKQ0Ize4qItWhUGhkSkqcBXm7eG/RFv66ZAuF+47QunkCF5/amSuHdOPMHh2Ji9OFciJSPoVCI1Zc4sxet4PJCzbx4ZItHDhaTNe2SVx+RjeuOqMbfdKSwy5RRBoYhUITcehoMR8v38pbCzYxffV2ikucU7u14YrB3bhscFdSk5PCLlFEGgCFQhNUuO8I7y3azFsLNrFk0x7i44zRvTtx1ZBuXHhKZ40/iDRhCoUmbk3BPibP38Q7CzezafchkpMSuGxQV67LzuD09LaaqE+kiVEoCBAZoP5q/Q7eyMnng6VbOHyshH5pyVybnc6VZ3Sjoy6QE2kSFAryD/YePsb7i7bwWk4eC/N2kxBnnDcgjeuGpTOmT4puFiTSiCkUpFJfb9vHa3PzeGvBJnYcOEpqcnOuHprOtUPT6ZnSOuzyRKSOKRSkWo4WlfDZygJez8nj81UFlDgM79GBm0ZkctGpnWmeoMFpkcZAoSA1tm3vYd6cn8+rc/LI3XmQ9i0TuWZoOhOGZ6r3IBLjFApSayUlzsy123l5di6fLN9GUYkzsmdHbhyRyYWndNYd5ERikEJB6kTB3sO8Pi+fV+bkkr/rEB1bNeOa7HRuHJ5J946twi5PRKpJoSB1qqTEmba6kJdn5/K3lQUUlzije3fixhGZnD8wjUSduSTSoCkUJGq27jnMazl5vDonl817DpOa3JwbR2Ry4/BMUttoWg2Rhij0UDCzDOB5IA1w4El3/2057cYBvwESge3uPray9SoUGo7iEueLrwt4YdZGPl9VSEKccdGpnbl1VBbZ3dvrqmmRBqQhhEIXoIu7zzezZGAecIW7Ly/Tph3wJXCRu+eaWaq7F1S2XoVCw7Rh+wFe/Gojr+XksfdwEf07J3PrqCwuH9yVls10K3CRsIUeCv/wRWbvAI+6+ydllt0NdHX3B6q7HoVCw3bwaBHvLNzMn7/cwMqt+2iTlMB12Rl868zuZHXSwLRIWBpUKJhZFjANONXd95ZZfvyw0SlAMvBbd3++nM9PBCYCZGZmDt24cWPUa5aT4+7kbNzFn7/cwEdLt1Lszti+Kdw6MouxfVN0QyCRetZgQsHMWgNfAP/l7pNPeO9RIBs4F2gBzAIudfevK1qfegqxp2DvYV6ek8tLs3Mp3HeE7h1bcvOZ3bluWAZtdL9pkXrRIELBzBKB94Ep7v5IOe/fB7Rw958Fr58BPnL31ytap0Ihdh0tKmHKsq08P2sDczfsolWzeK7NzuD2s7J0zYNIlIUeChY59eTPwE53v7eCNgOAR4ELgWbAHOAGd19a0XoVCo3D0k17eHbGet5bvJmiEue8AWl8Z3QPRvTooLOWRKKgIYTCaGA6sAQoCRbfD2QCuPsfgnb/CtwetHna3X9T2XoVCo3Ltr2HefGrjbz41UZ2HTzGKV3b8O2zevCNQV00GZ9IHQo9FKJFodA4HT5WzFsLNvHsjPWsLthPSnJzbj6zOzeNyNSNgETqgEJBYpK7M331dp6duZ6pqwpplhDHlYO78e3RPejXOTns8kRiVnVDQVcVSYNiZozpm8KYvimsKdjHszM3MHl+Pn/JyWNcvxQmje2lcQeRKFJPQRq8XQeO8uJXG3nuyw3sOHCUQRntmDSmJxec0pl4Xe8gUi06fCSNzuFjxbwxL5+npq9j446DZHVsyZ1jenL1kHSSEjUoLVIZhYI0WsUlzpRlW/nDF2tZnL+HTq2bcftZPfjWiO60bamL4UTKo1CQRs/dmbVuB3/8Yh1ffF1Iy2bxTBieyZ1n96RzW03hLVKWQkGalOWb9/LU9HW8u2gz8XHGDcMymDS2F13btQi7NJEGQaEgTVLezoM8PnUNr+fkE2fGtdnp3D2+N90UDtLEKRSkScvfdZAnpq7ltZw8AK4Zms7d43qT0aFlyJWJhEOhIAJs3n2IJ6au5S9z8yhx57phGdx7bh/dNlSaHIWCSBlb9xzm8alreHl2Lonxcdx5dg8mju1F6+a6flOaBoWCSDk2bD/AQx+v4q+Lt9CxVTO+f24fJgzPpFlCXNiliURVdUNB/xKkScnq1IrHbhzCO/ecRd+0ZH727jIu+d10vly7PezSRBoEhYI0SYMy2vHynSN45tZsjhQVc+NTs/nBqwso3Hck7NJEQqVQkCbLzDh3QBqf/HAs3z+nNx8u2cqFv5nGp8u3hV2aSGgUCtLkJSXG86ML+vHX74+mc5sk7ng+h/vfWsLBo0VhlyZS7xQKIoE+acm8dc8ovjumJ6/MyeWqx79k8+5DYZclUq8UCiJlNE+I56eXDOBPtw0jf9chrnhsJks37Qm7LJF6o1AQKce4fqm8edcoEuPjuOnp2azYsjfskkTqhUJBpAL9Oifz6sQzaZEYz83PzGZt4f6wSxKJuqiFgpllmNnnZrbczJaZ2Q8qaTvMzIrM7Jpo1SNSGxkdWvLSnSMA+PZzc9lz6FjIFYlEVzR7CkXAP7v7QOBM4B4zG3hiIzOLB34JfBzFWkRqrVdKa/5481A27TrEP7+2iJKS2JoFQKQmohYK7r7F3ecHz/cBK4Bu5TT9HvAmUBCtWkRO1tDuHfi3Swfw6Ypt/GHa2rDLEYmaehlTMLMs4Axg9gnLuwFXAk9U8fmJZpZjZjmFhYXRKlOkUreNyuIbp3fh11NWMWvtjrDLEYmKqIeCmbUm0hO4191PPIXjN8BP3L2ksnW4+5Punu3u2SkpKdEqVaRSZsaDV59OVqdWfO+VBRTsOxx2SSJ1LqqhYGaJRALhJXefXE6TbOBVM9sAXAM8bmZXRLMmkZPRunkCj980hP1HjnH3i/M5dLQ47JJE6lQ0zz4y4Blghbs/Ul4bd+/h7lnungW8Adzt7m9HqyaRutC/cxsevnYw83J3ccfzc9l7WGckSeMRzZ7CWcDNwDlmtjB4XGJmk8xsUhS/VyTqLj29C7++ZhCz1+3kst/P4Kt1GmOQxkE32RE5CbPX7eBf31hM7s6DXDM0ne+f04fMjroPtDQ8usmOSD0Y0bMjU+4dw3fH9OTdRZsZ//BUfvTaQtYU7Au7NJFaUU9BpI5s23uYp6at46XZuRw6VszZfTpx85ndOXdAGvFxFnZ50sTpHs0iIdmx/wivzMnlpdm5bNlzmG7tWnDjiExuGJZBx9bNwy5PmiiFgkjIiopL+HTFNp6ftZEv1+6gWXwcF5ySxoThmYzs2ZE49R6kHlU3FBLqoxiRpighPo6LTu3CRad2YU3BPl6encfkBfm8v3gLmR1acv2wDK4dmk5qm6SwSxUppZ6CSD06fKyYKcu28uqcPGat20F8nHFO/1QmDM9gbN9UjT1I1KinINIAJSXGc/ngblw+uBvrtx/gL3PzeGNePp8s30aXtklcm53BddnppLfXaa0SDvUUREJ2rLiEv63Yxitz8pi2OjLh49l9UpgwLIPzBqaRGK8zx+XkaaBZJAbl7zrIazn5vJ6Tx5Y9h+nUuhlXD03nhmGZ9OjUKuzyJIYpFERiWHGJ88XXBbwyJ4/PVhZQXOKM7NmRCSMyufCUNJonxIddosSYkw4FM3sPqCgxjgBrgcfcPa/WVdaCQkGamm17D/PGvHxenZtL3s5DtG+ZyNVD0rlheCa9U1uHXZ7EiLoIhbGVfC4BOAWY4O4ja1di7SgUpKkqKXFmrt3OK3Ny+XjZNopKnOE9OnDj8EwuOrUzSYnqPUjF6uXwkZk97e531HoFtaBQEIHCfUdKew8bdxykXctErjojnQnDM+iTlhx2edIAaUxBpAkoKXFmrdvBy3Ny+XjZVo4VO8Oy2nPDsEwuPb2Leg9SSqEg0sRs33+EyfPzeWVOHuu3H6BNUgJXDUlnwvBM+nVW76Gpq/NQMLOW7n7wpCs7SQoFkcq5O1+t28krc3L5aOlWjhaXMCSzHROGZ/KN07vSopl6D01RnYWCmY0CngZau3ummQ0Cvuvud9dNqTWjUBCpvp0HjjJ5fj4vz8llXeEBkpMSuGJwN246M5P+nduEXZ7Uo7oMhdnANcC77n5GsGypu59aJ5XWkEJBpObcnTnrI72HD5Zu5WhRCcOzOvCtkd256JTONEvQVdONXZ3OfeTueWZ/N1FXcW0LE5H6Z2aM6NmRET078rMDR3l9Xh4vfpXL919ZQKfWzZkwPIMbR2TSpW2LsEuVkFUnFPKCQ0huZonAD4AV0S1LRKKlfatmTBzTiztG9+SL1YW8MGsjj36+hsenruX8AWncPLI7o3p15IQ/BKWJqE4oTAJ+C3QDNgEfA/dU9SEzywCeB9KIXBn9pLv/9oQ2NwE/AQzYB9zl7otqsgEiUjtxccb4fqmM75dK3s6DvDh7I6/NzeOjZVvpldKKm8/szlVD02mTlBh2qVKPonZKqpl1Abq4+3wzSwbmAVe4+/IybUYBK9x9l5ldDPzc3UdUtl6NKYhEz+Fjxfx18Rae/2oji/J207JZPFec0Y1bRnbXwHSMq8uB5t+Vs3gPkOPu79SgoHeAR939kwrebw8sdfdula1HoSBSPxbn7+aFWRt5d9FmjhSVcFbvjnxndA/G9U3VrURjUF2GwpNAf+D1YNHVwHqgI7DO3e+tRjFZwDTgVHffW0GbfwH6lzdthplNBCYCZGZmDt24cWNVXykidWTXgaO8MjeX57/cyNa9h+mZ0orbz+rB1UO60bKZ7tMVK+oyFL4CznL34uB1AjAdGA0scfeBVXy+NfAF8F/uPrmCNuOBx4HR7r6jsvWppyASjmPFJXywZAvPzFjP4vw9tG2RyI0jMrl1ZBad2+o+0w1dXZ6S2h5oTeSQEUAroIO7F5vZkSqKSATeBF6qJBBOJ3Jx3MVVBYKIhCcxPo7LB3fjskFdydm4i2emr+ePX6zlqWnruPT0LtwxuienpbcNu0w5SdUJhV8BC81sKpGzhMYAvzCzVsCnFX3IIuezPUNkIPmRCtpkApOBm9396xrWLiIhMDOGZXVgWFYH8nYe5E8zN/BaTh7vLNzMWb07ctfY3pzVW6e0xqpqnX0UnEk0PHg51903V+Mzo4kcZloClASL7wcyAdz9D2b2NJExiuODBEVVdW90+Eik4dl7+BivzM7l6RnrKdx3hNPT23LX2F5ceEpnDUo3EHU6IV5wZlAfoPTAobtPO6kKa0mhINJwHT5WzFsLNvHHL9ayYcdBeqa0YtKYXlxxRjdNpRGyuhxovoPIVczpwELgTGCWu59TF4XWlEJBpOErLnE+XLqFJ6auZdnmvXRuk8Q943tx3bAM3V86JNUNhepE9w+AYcBGdx8PnAHsPsn6RKQRi48zvnF6V97/3mie//Zw0tu34N/fWcb4h6by0uyNHC0qqXolEorqhMJhdz8MYGbN3X0l0C+6ZYlIY2BmjOmbwuuTRvLCd4bTuW0S//bWUsb/eiqvzsnlWLHCoaGpTijkm1k74G3gk+DKZF09JiLVZmac3SeFN+8axXO3D6NTcnPum7yEcx6eyjsLN1FSElt3gGzMajT3kZmNBdoCH7n70ahVVQmNKYjEPnfn81UF/HrK1yzfspfT09vy04sHMLJXx7BLa7TqZKDZzOKBZe7evy6LOxkKBZHGo6TEeXvhJn49ZRWb9xzm3P6p3Hdxf/qk6Z7Sda1OBpqDqS1WBReZiYjUqbg446oh6Xz2L+O47+L+zNmwkwt/M42fv7uMvYePhV1ek1TdaS6Wmdkc4MDxhe5+WdSqEpEmJSkxnklje3Fddgb/88nXPD9rA+8v3sL9l/TnyjO66eroelSd6xTGlrfc3b+ISkVV0OEjkcZv6aY9PPD2Uhbm7WZ4Vgd+cdVp9E5tHXZZMa3OrlMI/ue/AUgMns8F5p90hSIiFTi1W1sm3zWKX159Gl8X7OOS303nqWnrKNZZSlFXZSiY2Z3AG8Afg0XdiJyeKiISNXFxxvXDMvn4h2MY2zeF//pgBdf/cRbrtx+o+sNSa9W5TuEe4CxgL4C7rwZSo1mUiMhxqclJPHnzUB65bhCrtu3j4t9O49U5uWGX1WhVJxSOlL0mIbjJjvpwIlJvzCJnKX3yw7EM7d6e+yYv4cdvLOLwseKwS2t0qhMKX5jZ/UALMzufyG0534tuWSIi/6hz2ySe//YIvndOb17Lyeeqx78kb+fBsMtqVKoTCvcBhUTui/Bd4APggWgWJSJSkfg4458v6Mezt2WTv+sgVz7+JUvy91T9QamW6oTCFcDz7n6tu1/j7k95TebGEBGJgnP6pzH57lE0T4jj+idnMWP19rBLahSqEwrfBL42sxfM7BvBmIKISOh6pyYz+e5RZHZoyXf+PJdZa3Wb95NVnesUbgd6ExlLmACsDW6jKSISurQ2Sbx0x4jSYJi3cWfYJcW0at0fz92PAR8CrwLziBxSEhFpEDq2bs5Ld44gNbk5E5+fx+bdh8IuKWZV5+K1i83sOWA1cDXwNNA5ynWJiNRIanIST9+azZGiEr77wjydrlpL1ekp3ELkCuZ+7n6bu3/g7kVVfcjMMszsczNbbmbLzOwH5bQxM/udma0xs8VmNqQW2yAiAkTGGP7n+sEs2bSH/3h/edjlxKTqjClMcPe33f0IgJmNNrPHqrHuIuCf3X0gcCZwj5kNPKHNxUCf4DEReKJG1YuInOD8gWncMboHL83OZfY6DTzXVLXGFMzsDDN7yMw2AP8BrKzqM+6+xd3nB8/3ASuIzJtU1uVETnd1d/8KaGdmXWqyASIiJ/rRBX3J6NCCn05eosNINVRhKJhZXzP7mZmtBH4P5BKZanu8u/++Jl9iZlnAGcDsE97qBuSVeZ3PPwYHZjbRzHLMLKewsLAmXy0iTVDLZgn84srTWLf9AI99vibscmJKZT2FlcA5wDfcfXQQBDWOXDNrDbwJ3Ovue2tTpLs/6e7Z7p6dkpJSm1WISBNzdp8ULh/clT9OW0f+Lk2FUV2VhcJVwBbgczN7yszOBWp0+yMzSyQSCC+5++RymmwCMsq8Tg+WiYictB9f1B8DfvXRqrBLiRkVhkIwuHwD0B/4HLgXSDWzJ8zsgqpWbJH75z0DrHD3Rypo9i5wS3AW0pnAHnffUuOtEBEpR7d2LZg4pifvLtrM/NxdYZcTE6pz9tEBd3/Z3b9J5C/5BcBPqrHus4CbgXPMbGHwuMTMJpnZpKDNB8A6YA3wFHB3rbZCRKQCk8b2IiW5Of/5/nI0bVvVajSPkbvvAp4MHlW1nUEVh5uCifXuqUkNIiI10ap5Av96QT9+/OZi3l+8hW8O6hp2SQ1atU5JFRGJZVcPTWdglzY8+OFKnaJaBYWCiDR68XHGA5cOYNPuQzw9fV3Y5TRoCgURaRJG9e7EJad15rd/W80CDTpXSKEgIk3Gf195Omltkvinlxew++DRqj/QBCkURKTJaNsykcduHELBvsPc+5eFFJfobKQTKRREpEkZlNGOn192ClNXFfLghyvCLqfB0a01RaTJuWlEd1Zv289T09fTO7U11w/LDLukBkM9BRFpkh64dABn9+nEA28v1RTbZSgURKRJSoiP49Ebh5DRoSWTXpxH7g5NmgcKBRFpwtq2SOSZW4dR4jDxhRxd2IZCQUSauB6dWvGbGwazcus+fvlRlfcPa/QUCiLS5I3vl8pto7L408wNTF1VEHY5oVIoiIgA913cn35pyfz4jcXsP1IUdjmhUSiIiABJifE8ePVpFOw7wu8/Wx12OaFRKIiIBM7IbM+1Q9N5dsZ61hbuD7ucUCgURETK+PFF/UlKiOf/vbc87FJCoVAQESkjJbk5/3ROb774upB5G5vebKoKBRGRE3zrzO50aNWsSY4tKBRERE7QqnkCd5zdg6mrClmUtzvscuqVQkFEpBy3jMyibYvEJtdbiFoomNmzZlZgZksreL+tmb1nZovMbJmZ3R6tWkREaqp18wRuPyuLT1cUsKZgX9jl1Jto9hSeAy6q5P17gOXuPggYBzxsZs2iWI+ISI3cfGZ3mifE8cyM9WGXUm+iFgruPg3YWVkTINnMDGgdtG26lxGKSIPTsXVzrhqSzpvzN7F9/5Gwy6kXYY4pPAoMADYDS4AfuHtJeQ3NbKKZ5ZhZTmFhYX3WKCJN3HdG9+BoUQkvfrUx7FLqRZihcCGwEOgKDAYeNbM25TV09yfdPdvds1NSUuqzRhFp4nqntuac/qm8MGtjk5haO8xQuB2Y7BFrgPVA/xDrEREp1x1n92DHgaO8vWBT2KVEXZihkAucC2BmaUA/YF2I9YiIlGtkz44M7NKGp2esp6TEwy4nqqJ5SuorwCygn5nlm9l3zGySmU0KmvwHMMrMlgB/A37i7tujVY+ISG2ZGXeO6cGagv18sbpxj2smRGvF7j6hivc3AxdE6/tFROrSpad15cEPV/LM9PWM75cadjlRoyuaRUSqoVlCHLeN6sGMNdtZvnlv2OVEjUJBRKSabhyeSctm8Y36YjaFgohINbVtmch12Rm8u2gTuTsOhl1OVCgURERq4K5xvUiIi+NXU1aGXUpUKBRERGogrU0Sd57dg/cXb2FhI5xWW6EgIlJDE8f2olPrZvzirytwb1zXLSgURERqqHXzBH50fj/mbNjJq3Pzwi6nTikURERq4YZhGZzVuyP/+f5y8nY2nkFnhYKISC3ExRm/umYQcWb8y+uLGs30FwoFEZFa6tauBf/nmwOZvX4nz85sHNcuKBRERE7CNUPTOW9AKr+asoq1hfvDLuekKRRERE6CmfGLq04jKSGOn765JOYPIykUREROUmpyEv/+jYHM2bCTl+bkhl3OSVEoiIjUgWuGpnN2n048+MEKCvYeDrucWlMoiIjUATPjP684laPFJfzPp6vDLqfWFAoiInWke8dW3DSiO6/l5LGmIDYHnRUKIiJ16Hvn9KZFYjwPxeiEeQoFEZE61LF1cyaO6cmUZdtYFIMT5ikURETq2LdH96BNUgKPT10Tdik1plAQEaljrZsncNuoLKYs28bqbfvCLqdGohYKZvasmRWY2dJK2owzs4VmtszMvohWLSIi9e22s3rQIjGeJ6auDbuUGolmT+E54KKK3jSzdsDjwGXufgpwbRRrERGpVx1aNWPC8EzeWbQ5pmZRjVoouPs0YGclTW4EJrt7btC+IFq1iIiE4c4xPYgzeHLaurBLqbYwxxT6Au3NbKqZzTOzWypqaGYTzSzHzHIKCwvrsUQRkdrr0rYFVw9J5y85eRTsi42rnMMMhQRgKHApcCHw72bWt7yG7v6ku2e7e3ZKSkp91igiclK+O7YXRcUlPDtjQ9ilVEuYoZAPTHH3A+6+HZgGDAqxHhGROtejUysuOa0LL37GvJ6YAAAKnklEQVS1kT2HjoVdTpXCDIV3gNFmlmBmLYERwIoQ6xERiYq7x/Vm/5EiXpi1IexSqhTNU1JfAWYB/cws38y+Y2aTzGwSgLuvAD4CFgNzgKfdvcLTV0VEYtXArm04p38qz87cwMGjRWGXU6mEaK3Y3SdUo81DwEPRqkFEpKG4Z3wvrn5iFn+auYF7xvcOu5wK6YpmEZF6MLR7B84bkMYTU9eyff+RsMupkEJBRKSe3Hdxfw4dK+Z3f2u491tQKIiI1JPeqa25YVgGL8/ObbD3W1AoiIjUo3vP60vLZvHc/9YSSko87HL+gUJBRKQepSQ354FLBzJn/U5enZsXdjn/QKEgIlLPrs1OZ2TPjvz3ByvYsudQ2OX8HYWCiEg9MzP++6rTKHbnrhfnc6SoOOySSikURERCkNWpFQ9fO4iFebv5p5cXcLSoJOySAIWCiEhoLj6tC//v8lP4ZPk2bnl2NjsPHA27JIWCiEiYbhmZxSPXDWJ+7m4ue3QGi/J2h1qPQkFEJGRXDUnn9e+OpLjEueqJL3loysrQxhkUCiIiDcCgjHZ8dO8YrjqjG499vpbLfj+TJfl76r0OhYKISAPRtkUiD107iD/dNozdh45yxeMzefjjVfU6CK1QEBFpYMb3T+Xje8dyxeBu/P6zNVz26AyWbqqfXoNCQUSkAWrbMpGHrxvEM7dms/PAUS5/bCbPzFgf9e9VKIiINGDnDkjjkx+O5fJBXenRqWXUvy9qN9kREZG60bZlIo9cP7hevks9BRERKaVQEBGRUgoFEREpFbVQMLNnzazAzJZW0W6YmRWZ2TXRqkVERKonmj2F54CLKmtgZvHAL4GPo1iHiIhUU9RCwd2nATuraPY94E2gIFp1iIhI9YU2pmBm3YArgSeq0XaimeWYWU5hYWH0ixMRaaLCHGj+DfATd69yUg93f9Lds909OyUlpR5KExFpmszdo7dysyzgfXc/tZz31gMWvOwEHAQmuvvbVayzENhYy5I6Adtr+dmGRtvSMGlbGiZtC3R39yr/qg7timZ373H8uZk9RyQ8Kg2E4HO17iqYWY67Z9f28w2JtqVh0rY0TNqW6otaKJjZK8A4oJOZ5QM/AxIB3P0P0fpeERGpvaiFgrtPqEHb26JVh4iIVF9Tu6L5ybALqEPaloZJ29IwaVuqKaoDzSIiEluaWk9BREQq0WRCwcwuMrNVZrbGzO4Lu56aMrMNZrbEzBaaWU6wrIOZfWJmq4Of7cOuszzlzYNVUe0W8btgPy02syHhVf6PKtiWn5vZpmDfLDSzS8q899NgW1aZ2YXhVP2PzCzDzD43s+VmtszMfhAsj7n9Usm2xOJ+STKzOWa2KNiW/xss72Fms4Oa/2JmzYLlzYPXa4L3s066CHdv9A8gHlgL9ASaAYuAgWHXVcNt2AB0OmHZr4D7guf3Ab8Mu84Kah8DDAGWVlU7cAnwIZFrWM4EZoddfzW25efAv5TTdmDwu9Yc6BH8DsaHvQ1BbV2AIcHzZODroN6Y2y+VbEss7hcDWgfPE4HZwX/v14AbguV/AO4Knt8N/CF4fgPwl5Otoan0FIYDa9x9nbsfBV4FLg+5prpwOfDn4PmfgStCrKVCXv48WBXVfjnwvEd8BbQzsy71U2nVKtiWilwOvOruR9x9PbCGyO9i6Nx9i7vPD57vA1YA3YjB/VLJtlSkIe8Xd/f9wcvE4OHAOcAbwfIT98vx/fUGcK6ZHb8ouFaaSih0A/LKvM6n8l+ahsiBj81snplNDJalufuW4PlWIC2c0mqlotpjdV/9U3BY5dkyh/FiYluCQw5nEPmrNKb3ywnbAjG4X8ws3swWEpko9BMiPZnd7l4UNClbb+m2BO/vATqezPc3lVBoDEa7+xDgYuAeMxtT9k2P9B9j8lSyWK498ATQCxgMbAEeDrec6jOz1kRmKr7X3feWfS/W9ks52xKT+8Xdi919MJBOpAfTvz6/v6mEwiYgo8zr9GBZzHD3TcHPAuAtIr8s24534YOfsTQFeUW1x9y+cvdtwT/kEuAp/vdQRIPeFjNLJPI/0ZfcfXKwOCb3S3nbEqv75Th33w18Dowkcrju+MXGZest3Zbg/bbAjpP53qYSCnOBPsEIfjMiAzLvhlxTtZlZKzNLPv4cuABYSmQbbg2a3Qq8E06FtVJR7e8CtwRnu5wJ7ClzOKNBOuHY+pVE9g1EtuWG4AyRHkAfYE5911ee4LjzM8AKd3+kzFsxt18q2pYY3S8pZtYueN4COJ/IGMnnwPG7U564X47vr2uAz4IeXu2FPdpeXw8iZ098TeT43L+FXU8Na+9J5GyJRcCy4/UTOXb4N2A18CnQIexaK6j/FSLd92NEjod+p6LaiZx98Viwn5YA2WHXX41teSGodXHwj7RLmfb/FmzLKuDisOsvU9doIoeGFgMLg8clsbhfKtmWWNwvpwMLgpqXAv8nWN6TSHCtAV4HmgfLk4LXa4L3e55sDbqiWURESjWVw0ciIlINCgURESmlUBARkVIKBRERKaVQEBGRUgoFiQlmtj/4mWVmN9bxuu8/4fWXdbn+umZmt5nZo2HXIY2TQkFiTRZQo1AocyVoRf4uFNx9VA1riilmFh92DdJwKRQk1jwInB3Mj//DYPKwh8xsbjDx2XcBzGycmU03s3eB5cGyt4MJBZcdn1TQzB4EWgTreylYdrxXYsG6l1rkXhbXl1n3VDN7w8xWmtlL5c1MGbT5ZTA//tdmdnaw/O/+0jez981s3PHvDr5zmZl9ambDg/WsM7PLyqw+I1i+2sx+VmZd3wq+b6GZ/fF4AATrfdjMFhGZNkGkfGFfwaeHHtV5APuDn+OA98ssnwg8EDxvDuQQmSN/HHAA6FGm7fGrc1sQuVq0Y9l1l/NdVxOZpTKeyGyhuUTm7h9HZDbKdCJ/WM0iMmHhiTVPBR4Onl8CfBo8vw14tEy794FxwXMnuMKWyBxXHxOZPnkQsLDM57cQufr4+LZkAwOA94DEoN3jwC1l1ntd2PtRj4b/qKpbLdLQXQCcbmbH54VpS2Qum6PAHI/Ml3/c983syuB5RtCussnDRgOvuHsxkYnivgCGAXuDdecDBNMcZwEzylnH8Ynm5gVtqnIU+Ch4vgQ44u7HzGzJCZ//xN13BN8/Oai1CBgKzA06Li343wntiolMGCdSKYWCxDoDvufuU/5uYeRwzIETXp8HjHT3g2Y2lci8MbV1pMzzYir+t3SknDZF/P2h27J1HHP343PPlBz/vLuXnDA2cuL8NE7kv8Wf3f2n5dRxOAg3kUppTEFizT4it1w8bgpwVzB1MmbWN5hJ9kRtgV1BIPQncovD444d//wJpgPXB+MWKURuxVkXs2luAAabWZyZZVC7u36db5H7KbcgcheumUQmsrvGzFKh9H7L3eugXmlC1FOQWLMYKA4GTJ8DfkvksMr8YLC3kPJvS/oRMMnMVhCZGfOrMu89CSw2s/nuflOZ5W8RGZRdROQv8R+7+9YgVE7GTGA9kQHwFcD8WqxjDpHDQenAi+6eA2BmDxC5Q18ckZlc7wE2nmS90oRollQRESmlw0ciIlJKoSAiIqUUCiIiUkqhICIipRQKIiJSSqEgIiKlFAoiIlJKoSAiIqX+P5t85rSqFbSJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(avg_cost_func)\n",
    "plt.ylabel('Average J')\n",
    "plt.xlabel('Iteration number')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_y(W, b, X, n_layers):\n",
    "    m = X.shape[0]\n",
    "    y = np.zeros((m,))\n",
    "    for i in range(m):\n",
    "        h, z = feed_forward(X[i, :], W, b)\n",
    "        y[i] = np.argmax(h[n_layers])\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13.908205841446453"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "y_pred = predict_y(W, b, X_test, 3)\n",
    "accuracy_score(y_test, y_pred)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}