PlainAE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "step 0, training loss: -575.36\n",
      "step 500, training loss: -217.666\n",
      "step 1000, training loss: -209.651\n",
      "step 1500, training loss: -207.869\n",
      "step 2000, training loss: -206.921\n",
      "step 2500, training loss: -205.546\n",
      "step 3000, training loss: -209.716\n",
      "step 3500, training loss: -200.66\n",
      "step 4000, training loss: -211.552\n",
      "step 4500, training loss: -204.528\n",
      "step 5000, training loss: -198.955\n",
      "step 5500, training loss: -198.846\n",
      "step 6000, training loss: -184.67\n",
      "step 6500, training loss: -178.391\n",
      "step 7000, training loss: -178.737\n",
      "step 7500, training loss: -174.097\n",
      "step 8000, training loss: -171.984\n",
      "step 8500, training loss: -170.98\n",
      "step 9000, training loss: -172.832\n",
      "step 9500, training loss: -165.714\n",
      "step 10000, training loss: -172.642\n",
      "step 10500, training loss: -167.484\n",
      "step 11000, training loss: -157.408\n",
      "step 11500, training loss: -157.162\n",
      "step 12000, training loss: -156.741\n",
      "step 12500, training loss: -157.371\n",
      "step 13000, training loss: -149.388\n",
      "step 13500, training loss: -156\n",
      "step 14000, training loss: -157.614\n",
      "step 14500, training loss: -158.522\n",
      "step 15000, training loss: -155.11\n",
      "step 15500, training loss: -153.515\n",
      "step 16000, training loss: -150.165\n",
      "step 16500, training loss: -149.351\n",
      "step 17000, training loss: -149.412\n",
      "step 17500, training loss: -152.852\n",
      "step 18000, training loss: -150.115\n",
      "step 18500, training loss: -151.65\n",
      "step 19000, training loss: -144.028\n",
      "step 19500, training loss: -137.408\n",
      "step 20000, training loss: -148.059\n",
      "step 20500, training loss: -140.174\n",
      "step 21000, training loss: -148.915\n",
      "step 21500, training loss: -151.34\n",
      "step 22000, training loss: -142.126\n",
      "step 22500, training loss: -147.648\n",
      "step 23000, training loss: -148.38\n",
      "step 23500, training loss: -153.699\n",
      "step 24000, training loss: -135.613\n",
      "step 24500, training loss: -138.623\n",
      "step 25000, training loss: -143.443\n",
      "step 25500, training loss: -144.431\n",
      "step 26000, training loss: -143.33\n",
      "step 26500, training loss: -140.687\n",
      "step 27000, training loss: -147.25\n",
      "step 27500, training loss: -141.929\n",
      "step 28000, training loss: -135.744\n",
      "step 28500, training loss: -139.618\n",
      "step 29000, training loss: -136.441\n",
      "step 29500, training loss: -138.431\n",
      "step 30000, training loss: -144.871\n",
      "step 30500, training loss: -137.635\n",
      "step 31000, training loss: -151.549\n",
      "step 31500, training loss: -141.703\n",
      "step 32000, training loss: -138.352\n",
      "step 32500, training loss: -144.75\n",
      "step 33000, training loss: -142.917\n",
      "step 33500, training loss: -141.708\n",
      "step 34000, training loss: -144.976\n",
      "step 34500, training loss: -140.157\n",
      "step 35000, training loss: -143.005\n",
      "step 35500, training loss: -137.758\n",
      "step 36000, training loss: -143.832\n",
      "step 36500, training loss: -137.677\n",
      "step 37000, training loss: -140.603\n",
      "step 37500, training loss: -132.183\n",
      "step 38000, training loss: -131.583\n",
      "step 38500, training loss: -139.695\n",
      "step 39000, training loss: -149.044\n",
      "step 39500, training loss: -139.302\n",
      "step 40000, training loss: -140.594\n",
      "step 40500, training loss: -139.37\n",
      "step 41000, training loss: -140.397\n",
      "step 41500, training loss: -139.183\n",
      "step 42000, training loss: -134.718\n",
      "step 42500, training loss: -142.427\n",
      "step 43000, training loss: -139.831\n",
      "step 43500, training loss: -145.817\n",
      "step 44000, training loss: -141.948\n",
      "step 44500, training loss: -136.608\n",
      "step 45000, training loss: -136.449\n",
      "step 45500, training loss: -144.332\n",
      "step 46000, training loss: -140.196\n",
      "step 46500, training loss: -142.07\n",
      "step 47000, training loss: -144.395\n",
      "step 47500, training loss: -133.999\n",
      "step 48000, training loss: -132.118\n",
      "step 48500, training loss: -134.811\n",
      "step 49000, training loss: -133.865\n",
      "step 49500, training loss: -136.711\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "\n",
    "\"\"\"TensorFlow MNIST AutoEncoder\n",
    "\n",
    "This is my attempt to write the autoencoder for MNIST by Andrej Karpathy using \n",
    "ConvNetJS in TensorFlow. Mostly to get some more experience working in \n",
    "Tensorflow.\n",
    "\n",
    "Sources:\n",
    "    - http://cs.stanford.edu/people/karpathy/convnetjs/demo/autoencoder.html\n",
    "    - https://www.tensorflow.org/get_started/mnist/pros\n",
    "\n",
    "Author: Gertjan van den Burg\n",
    "Date: Thu Oct 26 16:49:29 CEST 2017\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "# from magenta.models.image_stylization.image_utils import form_image_grid\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "BATCH_SIZE = 128\n",
    "GRID_ROWS = 5\n",
    "GRID_COLS = 10\n",
    "USE_RELU = False\n",
    "\n",
    "\n",
    "def weight_variable(shape):\n",
    "    # From the mnist tutorial\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(0.1, shape=shape)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "\n",
    "def fc_layer(previous, input_size, output_size):\n",
    "    W = weight_variable([input_size, output_size])\n",
    "    b = bias_variable([output_size])\n",
    "    \n",
    "    tf.summary.histogram('weights', W)\n",
    "    tf.summary.histogram('biases', b)\n",
    "    \n",
    "    return tf.matmul(previous, W) + b\n",
    "\n",
    "def get_ll(target, output):\n",
    "    return tf.reduce_sum(\n",
    "        target * tf.log(output + 1e-10) + \\\n",
    "        (1 - target) * tf.log(1 - output + 1e-10),\n",
    "        reduction_indices=[-1]\n",
    "    )\n",
    "\n",
    "\n",
    "def autoencoder(x):\n",
    "    # first fully connected layer with 50 neurons using tanh activation\n",
    "    l1 = tf.nn.tanh(fc_layer(x, 28*28, 128))\n",
    "    tf.summary.histogram('outputs_l_1', l1)\n",
    "    # second fully connected layer with 50 neurons using tanh activation\n",
    "    l2 = tf.nn.tanh(fc_layer(l1, 128, 64))\n",
    "    tf.summary.histogram('outputs_l_2', l2)\n",
    "    # third fully connected layer with 2 neurons\n",
    "    l3 = fc_layer(l2, 64, 2)\n",
    "    tf.summary.histogram('outputs_l_3', l3)\n",
    "    # fourth fully connected layer with 50 neurons and tanh activation\n",
    "    l4 = tf.nn.tanh(fc_layer(l3, 2, 64))\n",
    "    tf.summary.histogram('outputs_l_4', l4)\n",
    "    # fifth fully connected layer with 50 neurons and tanh activation\n",
    "    l5 = tf.nn.tanh(fc_layer(l4, 64, 128))\n",
    "    tf.summary.histogram('outputs_l_5', l5)\n",
    "    # readout layer\n",
    "    if USE_RELU:\n",
    "        out = tf.nn.relu(fc_layer(l5, 128, 28*28))\n",
    "    else:\n",
    "        out = tf.sigmoid(fc_layer(l5, 128, 28*28))\n",
    "    tf.summary.histogram('outputs_l_6', out)\n",
    "    # let's use an l2 loss on the output image\n",
    "    loss = tf.reduce_mean(get_ll(x, out))\n",
    "    return loss, out, l3\n",
    "\n",
    "\"\"\"\n",
    "def layer_grid_summary(name, var, image_dims):\n",
    "    prod = np.prod(image_dims)\n",
    "    grid = form_image_grid(tf.reshape(var, [BATCH_SIZE, prod]), [GRID_ROWS, \n",
    "        GRID_COLS], image_dims, 1)\n",
    "    return tf.summary.image(name, grid)\n",
    "\"\"\"\n",
    "\n",
    "def create_summaries(loss, x, latent, output):\n",
    "    writer = tf.summary.FileWriter(\"./logs\")\n",
    "    tf.summary.scalar(\"Loss\", loss)\n",
    "    #layer_grid_summary(\"Input\", x, [28, 28])\n",
    "    #layer_grid_summary(\"Encoder\", latent, [2, 1])\n",
    "    #layer_grid_summary(\"Output\", output, [28, 28])\n",
    "    return writer, tf.summary.merge_all()\n",
    "\n",
    "\"\"\"\n",
    "def make_image(name, var, image_dims):\n",
    "    prod = np.prod(image_dims)\n",
    "    grid = form_image_grid(tf.reshape(var, [BATCH_SIZE, prod]), [GRID_ROWS, \n",
    "        GRID_COLS], image_dims, 1)\n",
    "    s_grid = tf.squeeze(grid, axis=0)\n",
    "\n",
    "    # This reproduces the code in: tensorflow/core/kernels/summary_image_op.cc\n",
    "    im_min = tf.reduce_min(s_grid)\n",
    "    im_max = tf.reduce_max(s_grid)\n",
    "\n",
    "    kZeroThreshold = tf.constant(1e-6)\n",
    "    max_val = tf.maximum(tf.abs(im_min), tf.abs(im_max))\n",
    "\n",
    "    offset = tf.cond(\n",
    "            im_min < tf.constant(0.0),\n",
    "            lambda: tf.constant(128.0),\n",
    "            lambda: tf.constant(0.0)\n",
    "            )\n",
    "    scale = tf.cond(\n",
    "            im_min < tf.constant(0.0),\n",
    "            lambda: tf.cond(\n",
    "                max_val < kZeroThreshold,\n",
    "                lambda: tf.constant(0.0),\n",
    "                lambda: tf.div(127.0, max_val)\n",
    "                ),\n",
    "            lambda: tf.cond(\n",
    "                im_max < kZeroThreshold,\n",
    "                lambda: tf.constant(0.0),\n",
    "                lambda: tf.div(255.0, im_max)\n",
    "                )\n",
    "            )\n",
    "    s_grid = tf.cast(tf.add(tf.multiply(s_grid, scale), offset), tf.uint8)\n",
    "    enc = tf.image.encode_jpeg(s_grid)\n",
    "\n",
    "    fwrite = tf.write_file(name, enc)\n",
    "    return fwrite\n",
    "\"\"\"\n",
    "\n",
    "def main():\n",
    "    # initialize the data\n",
    "    mnist = input_data.read_data_sets('/tmp/MNIST_data')\n",
    "\n",
    "    # placeholders for the images\n",
    "    x = tf.placeholder(tf.float32, shape=[None, 784])\n",
    "\n",
    "    # build the model\n",
    "    loss, output, latent = autoencoder(x)\n",
    "\n",
    "    # and we use the Adam Optimizer for training\n",
    "    optimizer = tf.train.AdamOptimizer(1e-4)\n",
    "    train_step = optimizer.minimize(-loss)\n",
    "\n",
    "    # We want to use Tensorboard to visualize some stuff\n",
    "    writer, summary_op = create_summaries(loss, x, latent, output)\n",
    "\n",
    "    first_batch = mnist.train.next_batch(BATCH_SIZE)\n",
    "\n",
    "    # Run the training loop\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        #sess.run(make_image(\"images/input.jpg\", x, [28, 28]), feed_dict={x : \n",
    "            #first_batch[0]})\n",
    "        for i in range(int(50000)):\n",
    "            batch = mnist.train.next_batch(BATCH_SIZE)\n",
    "            feed = {x : batch[0]}\n",
    "            if i % 500 == 0:\n",
    "                summary, train_loss = sess.run([summary_op, loss], \n",
    "                        feed_dict=feed)\n",
    "                print(\"step %d, training loss: %g\" % (i, train_loss))\n",
    "\n",
    "                writer.add_summary(summary, i)\n",
    "                writer.flush()\n",
    "\n",
    "            #if i % 1000 == 0:\n",
    "                #sess.run(make_image(\"images/output_%06i.jpg\" % i, output, [28, \n",
    "                #    28]), feed_dict={x : first_batch[0]})\n",
    "\n",
    "            train_step.run(feed_dict=feed)\n",
    "\n",
    "        # Save latent space\n",
    "        #pred = sess.run(latent, feed_dict={x : mnist.test._images})\n",
    "        #pred = np.asarray(pred)\n",
    "        #pred = np.reshape(pred, (mnist.test._num_examples, 2))\n",
    "        #labels = np.reshape(mnist.test._labels, (mnist.test._num_examples, 1))\n",
    "        #pred = np.hstack((pred, labels))\n",
    "        if USE_RELU:\n",
    "            fname = \"latent_relu.csv\"\n",
    "        else:\n",
    "            fname = \"latent_default.csv\"\n",
    "        #np.savetxt(fname, pred)\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "step 0, training loss: -577.485\n",
    "step 500, training loss: -202.786\n",
    "step 1000, training loss: -207.21\n",
    "step 1500, training loss: -202.004\n",
    "step 2000, training loss: -187.898\n",
    "step 2500, training loss: -186.689\n",
    "step 3000, training loss: -175.804\n",
    "step 3500, training loss: -172.61\n",
    "step 4000, training loss: -161.926\n",
    "step 4500, training loss: -170.936\n",
    "step 5000, training loss: -158.872\n",
    "step 5500, training loss: -164.085\n",
    "step 6000, training loss: -151.848\n",
    "step 6500, training loss: -159.407\n",
    "step 7000, training loss: -159.056\n",
    "step 7500, training loss: -157.305\n",
    "step 8000, training loss: -158.951\n",
    "step 8500, training loss: -161.98\n",
    "step 9000, training loss: -146.214\n",
    "step 9500, training loss: -154.979\n",
    "step 10000, training loss: -145.297\n",
    "step 10500, training loss: -143.071\n",
    "step 11000, training loss: -150.968\n",
    "step 11500, training loss: -157.205\n",
    "step 12000, training loss: -146.146\n",
    "step 12500, training loss: -144.243\n",
    "step 13000, training loss: -144.019\n",
    "step 13500, training loss: -163.697\n",
    "step 14000, training loss: -150.674\n",
    "step 14500, training loss: -152.286\n",
    "step 15000, training loss: -146.486\n",
    "step 15500, training loss: -157.113\n",
    "step 16000, training loss: -140.504\n",
    "step 16500, training loss: -136.147\n",
    "step 17000, training loss: -140.873\n",
    "step 17500, training loss: -144.909\n",
    "step 18000, training loss: -131.302\n",
    "step 18500, training loss: -147.198\n",
    "step 19000, training loss: -147.718\n",
    "step 19500, training loss: -145.261\n",
    "step 20000, training loss: -139.972\n",
    "step 20500, training loss: -140.979\n",
    "step 21000, training loss: -147.93\n",
    "step 21500, training loss: -140.522\n",
    "step 22000, training loss: -155.479\n",
    "step 22500, training loss: -147.228\n",
    "step 23000, training loss: -139.919\n",
    "step 23500, training loss: -132.262\n",
    "step 24000, training loss: -142.463\n",
    "step 24500, training loss: -130.251\n",
    "step 25000, training loss: -148.649\n",
    "step 25500, training loss: -141.536\n",
    "step 26000, training loss: -140.558\n",
    "step 26500, training loss: -137.23\n",
    "step 27000, training loss: -144.847\n",
    "step 27500, training loss: -135.864\n",
    "step 28000, training loss: -137.215\n",
    "step 28500, training loss: -128.116\n",
    "step 29000, training loss: -139.246\n",
    "step 29500, training loss: -131.523\n",
    "step 30000, training loss: -132.965\n",
    "step 30500, training loss: -137.978\n",
    "step 31000, training loss: -141.618\n",
    "step 31500, training loss: -140.294\n",
    "step 32000, training loss: -132.861\n",
    "step 32500, training loss: -130.252\n",
    "step 33000, training loss: -138.328\n",
    "step 33500, training loss: -142.932\n",
    "step 34000, training loss: -137.06\n",
    "step 34500, training loss: -134.373\n",
    "step 35000, training loss: -131.89\n",
    "step 35500, training loss: -132.643\n",
    "step 36000, training loss: -141.806\n",
    "step 36500, training loss: -147.092\n",
    "step 37000, training loss: -129.737\n",
    "step 37500, training loss: -138.656\n",
    "step 38000, training loss: -134.118\n",
    "step 38500, training loss: -139.848\n",
    "step 39000, training loss: -132.592\n",
    "step 39500, training loss: -125.842\n",
    "step 40000, training loss: -138.065\n",
    "step 40500, training loss: -143.349\n",
    "step 41000, training loss: -141.07\n",
    "step 41500, training loss: -137.043\n",
    "step 42000, training loss: -126.963\n",
    "step 42500, training loss: -136.094\n",
    "step 43000, training loss: -141.677\n",
    "step 43500, training loss: -129.368\n",
    "step 44000, training loss: -131.486\n",
    "step 44500, training loss: -134.046\n",
    "step 45000, training loss: -135.94\n",
    "step 45500, training loss: -131.42\n",
    "step 46000, training loss: -139.378\n",
    "step 46500, training loss: -133.611\n",
    "step 47000, training loss: -137.304\n",
    "step 47500, training loss: -142.816\n",
    "step 48000, training loss: -127.18\n",
    "step 48500, training loss: -129.672\n",
    "step 49000, training loss: -146.451\n",
    "step 49500, training loss: -138.04\n",
    "step 50000, training loss: -132.081\n",
    "step 50500, training loss: -130.092\n",
    "step 51000, training loss: -144.464\n",
    "step 51500, training loss: -139.677\n",
    "step 52000, training loss: -134.809\n",
    "step 52500, training loss: -139.341\n",
    "step 53000, training loss: -131.66\n",
    "step 53500, training loss: -136.415\n",
    "step 54000, training loss: -130.769\n",
    "step 54500, training loss: -133.07\n",
    "step 55000, training loss: -125.89\n",
    "step 55500, training loss: -143.225\n",
    "step 56000, training loss: -131.217\n",
    "step 56500, training loss: -143.6\n",
    "step 57000, training loss: -130.617\n",
    "step 57500, training loss: -143.195\n",
    "step 58000, training loss: -135.302\n",
    "step 58500, training loss: -141.117\n",
    "step 59000, training loss: -137.994\n",
    "step 59500, training loss: -123.757\n",
    "step 60000, training loss: -134.097\n",
    "step 60500, training loss: -134.512\n",
    "step 61000, training loss: -137.98\n",
    "step 61500, training loss: -137.607\n",
    "step 62000, training loss: -124.163\n",
    "step 62500, training loss: -136.144\n",
    "step 63000, training loss: -122.271\n",
    "step 63500, training loss: -129.956\n",
    "step 64000, training loss: -122.583\n",
    "step 64500, training loss: -131.837\n",
    "step 65000, training loss: -145.533\n",
    "step 65500, training loss: -134.17\n",
    "step 66000, training loss: -131.014\n",
    "step 66500, training loss: -128.104\n",
    "step 67000, training loss: -141.223\n",
    "step 67500, training loss: -120.874\n",
    "step 68000, training loss: -142.412\n",
    "step 68500, training loss: -128.993\n",
    "step 69000, training loss: -130.299\n",
    "step 69500, training loss: -120.395\n",
    "step 70000, training loss: -130.241\n",
    "step 70500, training loss: -128.016\n",
    "step 71000, training loss: -124.969\n",
    "step 71500, training loss: -129.618\n",
    "step 72000, training loss: -131.069\n",
    "step 72500, training loss: -140.832\n",
    "step 73000, training loss: -128.5\n",
    "step 73500, training loss: -135.724\n",
    "step 74000, training loss: -131.912\n",
    "step 74500, training loss: -137.808\n",
    "step 75000, training loss: -122.381\n",
    "step 75500, training loss: -134.815\n",
    "step 76000, training loss: -129.52\n",
    "step 76500, training loss: -137.294\n",
    "step 77000, training loss: -129.343\n",
    "step 77500, training loss: -137.771\n",
    "step 78000, training loss: -131.339\n",
    "step 78500, training loss: -126.507\n",
    "step 79000, training loss: -129.441\n",
    "step 79500, training loss: -136.201\n",
    "step 80000, training loss: -127.229\n",
    "step 80500, training loss: -129.502\n",
    "step 81000, training loss: -126.975\n",
    "step 81500, training loss: -130.243\n",
    "step 82000, training loss: -124.352\n",
    "step 82500, training loss: -133.099\n",
    "step 83000, training loss: -135\n",
    "step 83500, training loss: -128.062\n",
    "step 84000, training loss: -134.459\n",
    "step 84500, training loss: -124.606\n",
    "step 85000, training loss: -125.123\n",
    "step 85500, training loss: -123.388\n",
    "step 86000, training loss: -133.21\n",
    "step 86500, training loss: -129.994\n",
    "step 87000, training loss: -126.751\n",
    "step 87500, training loss: -128.521\n",
    "step 88000, training loss: -129.421\n",
    "step 88500, training loss: -134.652\n",
    "step 89000, training loss: -118.857\n",
    "step 89500, training loss: -127.451\n",
    "step 90000, training loss: -135.981\n",
    "step 90500, training loss: -128.619\n",
    "step 91000, training loss: -127.653\n",
    "step 91500, training loss: -131.686\n",
    "step 92000, training loss: -132.896"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}