11_convolutional_neural_networks_ex10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![book](https://raw.githubusercontent.com/ageron/tensorflow-safari-course/master/images/intro_to_tf_course.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Try not to peek at the solutions when you go through the exercises. ;-)**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's make sure this notebook works well in both Python 2 and Python 3:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"tmp/data/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model_params():\n",
    "    gvars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)\n",
    "    return {gvar.op.name: value for gvar, value in zip(gvars, tf.get_default_session().run(gvars))}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load demo image:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.misc import imread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "china = imread(\"./images/china.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "china.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_image(image):\n",
    "    cmap = \"gray\" if len(image.shape) == 2 else None\n",
    "    plt.imshow(image, cmap=cmap, interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plot_image(china)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Crop it and convert it to grayscale:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = china[150:220, 130:250].mean(axis=2).astype(np.float32)\n",
    "image.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "height, width = image.shape\n",
    "channels = 1  # grayscale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plot_image(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "basic_filters = np.zeros(shape=(7, 7, 1, 2), dtype=np.float32)  # height, width, in channels, out channels\n",
    "basic_filters[:, 3, 0, 0] = 1\n",
    "basic_filters[3, :, 0, 1] = 1\n",
    "plot_image(basic_filters[:, :, 0, 0])\n",
    "plt.show()\n",
    "plot_image(basic_filters[:, :, 0, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X = tf.placeholder(tf.float32, shape=(None, height, width, channels))\n",
    "    filters = tf.constant(basic_filters)\n",
    "    convolution = tf.nn.conv2d(X, filters, strides=[1,1,1,1], padding=\"SAME\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph) as sess:\n",
    "    X_batch = image.reshape(1, height, width, 1)\n",
    "    output = convolution.eval(feed_dict={X: X_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plot_image(output[0, :, :, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plot_image(output[0, :, :, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's add a max pooling layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X = tf.placeholder(tf.float32, shape=(None, height, width, channels))\n",
    "    filters = tf.constant(basic_filters)\n",
    "    convolution = tf.nn.conv2d(X, filters, strides=[1,1,1,1], padding=\"SAME\")\n",
    "    max_pool = tf.nn.max_pool(convolution, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding=\"VALID\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph) as sess:\n",
    "    X_batch = image.reshape(1, height, width, 1)\n",
    "    output = max_pool.eval(feed_dict={X: X_batch})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(5,3))\n",
    "plot_image(output[0, :, :, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(5,3))\n",
    "plot_image(output[0, :, :, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Exercise](https://c1.staticflickr.com/9/8101/8553474140_c50cf08708_b.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this final exercise, you will tackle MNIST and reach over 99% accuracy using most of what you learned in this course:\n",
    "\n",
    "* You model should be a Convolutional Neural Network composed of:\n",
    "  * Two convolutional layers followed by a max pooling layer. The first convolutional layer should have 32 feature maps, and the second should have 64 feature maps. Both convolutional layers should use ReLU activation, 3x3 filters, SAME padding and stride 1.\n",
    "  * One Fully Connected (FC) layer with 128 neurons, using ReLU activation.\n",
    "  * A Fully Connected output layer with 10 outputs (to classify images in the 10 classes), using Softmax activation.\n",
    "* You should apply a 25% dropout rate on the outputs of the max pooling layer, and a 50% dropout rate on the outputs of the first FC layer.\n",
    "* As usual, you should minimize the cross-entropy, using an Adam optimizer.\n",
    "* Make sure to initialize all variables using He initialization.\n",
    "* Train the model using Early Stopping.\n",
    "* Use the model to predict the class of all the images in the MNIST test set. Display all the wrong predictions it makes on the first 400 images, along with the probabilities it assigned to each class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try not to peek at the solution below before you have done the exercise! :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![thinking](https://upload.wikimedia.org/wikipedia/commons/0/06/Filos_segundo_logo_%28flipped%29.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise 10 - Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "height = 28\n",
    "width = 28\n",
    "channels = 1\n",
    "\n",
    "conv1_fmaps = 32\n",
    "conv1_ksize = 3\n",
    "conv1_stride = 1\n",
    "conv1_pad = \"SAME\"\n",
    "\n",
    "conv2_fmaps = 64\n",
    "conv2_ksize = 3\n",
    "conv2_stride = 1\n",
    "conv2_pad = \"SAME\"\n",
    "conv2_dropout_rate = 0.25\n",
    "\n",
    "pool3_fmaps = conv2_fmaps\n",
    "\n",
    "n_fc1 = 128\n",
    "fc1_dropout_rate = 0.5\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_outputs = 10\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.name_scope(\"inputs\"):\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs], name=\"X\")\n",
    "        X_reshaped = tf.reshape(X, shape=[-1, height, width, channels])\n",
    "        y = tf.placeholder(tf.int32, shape=[None], name=\"y\")\n",
    "        training = tf.placeholder_with_default(False, shape=[], name='training')\n",
    "\n",
    "    conv1 = tf.layers.conv2d(X_reshaped, conv1_fmaps, kernel_size=conv1_ksize, strides=conv1_stride, padding=conv1_pad, activation=tf.nn.relu, name=\"conv1\")\n",
    "    conv2 = tf.layers.conv2d(conv1, conv2_fmaps, kernel_size=conv2_ksize, strides=conv2_stride, padding=conv2_pad, activation=tf.nn.relu, name=\"conv2\")\n",
    "\n",
    "    with tf.name_scope(\"pool3\"):\n",
    "        pool3 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding=\"VALID\")\n",
    "        pool3_flat = tf.reshape(pool3, shape=[-1, pool3_fmaps * 14 * 14])\n",
    "        pool3_flat_drop = tf.layers.dropout(pool3_flat, conv2_dropout_rate, training=training)\n",
    "\n",
    "    with tf.name_scope(\"fc1\"):\n",
    "        fc1 = tf.layers.dense(pool3_flat_drop, n_fc1, activation=tf.nn.relu, name=\"fc1\")\n",
    "        fc1_drop = tf.layers.dropout(fc1, fc1_dropout_rate, training=training)\n",
    "\n",
    "    with tf.name_scope(\"output\"):\n",
    "        logits = tf.layers.dense(fc1_drop, n_outputs, name=\"output\")\n",
    "        Y_proba = tf.nn.softmax(logits, name=\"Y_proba\")\n",
    "\n",
    "    with tf.name_scope(\"train\"):\n",
    "        xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "        loss = tf.reduce_mean(xentropy)\n",
    "        optimizer = tf.train.AdamOptimizer()\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "    with tf.name_scope(\"eval\"):\n",
    "        correct = tf.nn.in_top_k(logits, y, 1)\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "    with tf.name_scope(\"init_and_save\"):\n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let training begin, using early stopping. This is quite slow on a CPU, but much faster on a GPU. We achieve >99% accuracy on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def restore_model_params(model_params):\n",
    "    gvar_names = list(model_params.keys())\n",
    "    assign_ops = {gvar_name: tf.get_default_graph().get_operation_by_name(gvar_name + \"/Assign\")\n",
    "                  for gvar_name in gvar_names}\n",
    "    init_values = {gvar_name: assign_op.inputs[1] for gvar_name, assign_op in assign_ops.items()}\n",
    "    feed_dict = {init_values[gvar_name]: model_params[gvar_name] for gvar_name in gvar_names}\n",
    "    tf.get_default_session().run(assign_ops, feed_dict=feed_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NB: The following cell will not run on the resources available for the course. If you are running locally, it is highly recommended to use a GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 1000\n",
    "batch_size = 50\n",
    "\n",
    "best_acc_val = 0\n",
    "check_interval = 100\n",
    "checks_since_last_progress = 0\n",
    "max_checks_without_progress = 100\n",
    "best_model_params = None\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for iteration in range(mnist.train.num_examples // batch_size):\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch, training: True})\n",
    "            if iteration % check_interval == 0:\n",
    "                acc_val = accuracy.eval(feed_dict={X: mnist.validation.images[:2000], y: mnist.validation.labels[:2000]})\n",
    "                if acc_val > best_acc_val:\n",
    "                    best_acc_val = acc_val\n",
    "                    checks_since_last_progress = 0\n",
    "                    best_model_params = get_model_params()\n",
    "                else:\n",
    "                    checks_since_last_progress += 1\n",
    "        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        acc_val = accuracy.eval(feed_dict={X: mnist.validation.images[2000:], y: mnist.validation.labels[2000:]})\n",
    "        print(epoch, \"Train accuracy:\", acc_train, \"Validation accuracy:\", acc_val, \"Best validation accuracy:\", best_acc_val)\n",
    "        if checks_since_last_progress > max_checks_without_progress:\n",
    "            print(\"Early stopping!\")\n",
    "            break\n",
    "\n",
    "    if best_model_params:\n",
    "        restore_model_params(best_model_params)\n",
    "    acc_test = accuracy.eval(feed_dict={X: mnist.test.images[2000:], y: mnist.test.labels[2000:]})\n",
    "    print(\"Final accuracy on test set:\", acc_test)\n",
    "    save_path = saver.save(sess, \"./my_mnist_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session(graph=graph) as sess:\n",
    "    init.run()\n",
    "    saver.restore(sess, \"./my_mnist_model\")\n",
    "    Y_proba_val = Y_proba.eval(feed_dict={X: mnist.test.images[2000:2400]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for image, y_label, y_proba in zip(mnist.test.images[2000:2400], mnist.test.labels[2000:2400], Y_proba_val):\n",
    "    y_pred = np.argmax(y_proba)\n",
    "    if y_pred != y_label:\n",
    "        print(\"Label: {}, Prediction: {}, Probabilities: {}\".format(\n",
    "            y_label, y_pred,\n",
    "            \"   \".join([\"{}={:.1f}%\".format(n, 100*p)\n",
    "                        for n, p in enumerate(y_proba) if p > 0.01])))\n",
    "        plt.imshow(image.reshape(28, 28), cmap=\"binary\")\n",
    "        plt.axis(\"off\")\n",
    "        plt.show()    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# What Next?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Practice, practice and practice!\n",
    "* Go through the nice tutorials on tensorflow.org, in particular the transfer learning one.\n",
    "* Buy [my book](http://shop.oreilly.com/product/0636920052289.do)! :)  There's a lot more material, including Recurrent Neural Networks, Deep Reinforcement Learning (including the amazing DeepMind stuff), Distributed TensorFlow, Autoencoders, and much more.\n",
    "* Go through the notebooks on my other Github project [github.com/ageron/handson-ml](https://github.com/ageron/handson-ml)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![book](http://akamaicovers.oreilly.com/images/0636920052289/cat.gif)](http://shop.oreilly.com/product/0636920052289.do)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I hope you enjoyed this course!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}