2-15 딥러닝3 복습.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2-15 딥러닝-3 복습.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/juhee3199/Machine-learning_advanced-study/blob/master/2-15%20%EB%94%A5%EB%9F%AC%EB%8B%9D3%20%EB%B3%B5%EC%8A%B5.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xFinVgoL_XQq"
      },
      "source": [
        "## 문제 1\n",
        "\n",
        "numerical_example_code.py 파일에는 수치적 미분으로 구현한\n",
        "\n",
        "수업시간 DNN 예제 코드가 저장되어 있습니다. 수치 미분을 이용한 구현은 명확한 한계점이 존재합니다. \n",
        "\n",
        "만약 hidden_depth=5를 10으로 2배로 늘린다면 예상되는 실행시간은 몇배로 증가할까요?\n",
        "\n",
        "(파라미터 대입을 통한 답이 아닌, 이론적으로 설명하세요)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0OTN2DXVffhC"
      },
      "source": [
        "\"\"\"수치 미분을 이용한 구현은 gradientdescent한 스텝을 계산을 위해 곱연산이 약 N*(N+1)번 필요하다.\r\n",
        "hiden_depth를 2배로 늘리면 파라미터의 수가 2N개가 되어서 곱연산이 2N*(2N+1)번 필요할 것이다. \r\n",
        "이에 따라 실행 시간은 약 3배로 증가한다.\r\n",
        "\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NOjJMS9O_XQ4",
        "scrolled": true
      },
      "source": [
        "### 수치 미분을 이용한 심층 신경망 학습\n",
        "\n",
        "# ## Import modules\n",
        "\n",
        "\n",
        "import time\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "# ##  예제 코드\n",
        "\n",
        "epsilon = 0.0001\n",
        "\n",
        "def _t(x):\n",
        "    return np.transpose(x)\n",
        "\n",
        "def _m(A, B):\n",
        "    return np.matmul(A, B)\n",
        "\n",
        "def sigmoid(x):\n",
        "    return 1 / (1 + np.exp(-x))\n",
        "\n",
        "def mean_squared_error(h, y):\n",
        "    return 1 / 2 * np.mean(np.square(h - y))\n",
        "\n",
        "class Dense:\n",
        "    def __init__(self, W, b, a):\n",
        "        self.W = W\n",
        "        self.b = b\n",
        "        self.a = a\n",
        "        \n",
        "        self.dW = np.zeros_like(self.W)\n",
        "        self.db = np.zeros_like(self.b)\n",
        "\n",
        "    def __call__(self, x):\n",
        "        return self.a(_m(_t(self.W), x) + self.b)   # matmul((ixo)T,ix1) + ox1\n",
        "\n",
        "\n",
        "class DNN:\n",
        "    def __init__(self, hidden_depth, num_neuron, num_input, num_output, activation=sigmoid):\n",
        "        def init_var(i, o):\n",
        "            return np.random.normal(0.0, 0.01, (i, o)), np.zeros((o,))\n",
        "\n",
        "        self.sequence = list()\n",
        "        # First hidden layer\n",
        "        W, b = init_var(num_input, num_neuron)\n",
        "        self.sequence.append(Dense(W, b, activation))\n",
        "        \n",
        "        # Hidden layers\n",
        "        for _ in range(hidden_depth - 1):\n",
        "            W, b = init_var(num_neuron, num_neuron)\n",
        "            self.sequence.append(Dense(W, b, activation))\n",
        "\n",
        "        # Output layer\n",
        "        W, b = init_var(num_neuron, num_output)\n",
        "        self.sequence.append(Dense(W, b, activation))\n",
        "\n",
        "    def __call__(self, x):\n",
        "        for layer in self.sequence:\n",
        "            x = layer(x)\n",
        "        return x\n",
        "\n",
        "    def calc_gradient(self, x, y, loss_func):\n",
        "        def get_new_sequence(layer_index, new_layer):\n",
        "            new_sequence = list()\n",
        "            for i, layer in enumerate(self.sequence):\n",
        "                if i == layer_index:\n",
        "                    new_sequence.append(new_layer)\n",
        "                else:\n",
        "                    new_sequence.append(layer)\n",
        "            return new_sequence\n",
        "        \n",
        "        def eval_sequence(x, sequence):\n",
        "            for layer in sequence:\n",
        "                x = layer(x)\n",
        "            return x\n",
        "        \n",
        "        loss = loss_func(self(x), y)\n",
        "        \n",
        "        for layer_id, layer in enumerate(self.sequence):\n",
        "            for w_i, w in enumerate(layer.W):\n",
        "                for w_j, ww in enumerate(w):\n",
        "                    W = np.copy(layer.W)\n",
        "                    W[w_i][w_j] = ww + epsilon\n",
        "                    \n",
        "                    new_layer = Dense(W, layer.b, layer.a)\n",
        "                    new_seq = get_new_sequence(layer_id, new_layer)\n",
        "                    h = eval_sequence(x, new_seq)\n",
        "                    \n",
        "                    num_grad = (loss_func(h, y) - loss) / epsilon # (f(x+eps) - f(x)) / eps\n",
        "                    layer.dW[w_i][w_j] = num_grad\n",
        "                    \n",
        "            for b_i, bb in enumerate(layer.b):\n",
        "                b = np.copy(layer.b)\n",
        "                b[b_i] = bb + epsilon\n",
        "\n",
        "                new_layer = Dense(layer.W, b, layer.a)\n",
        "                new_seq = get_new_sequence(layer_id, new_layer)\n",
        "                h = eval_sequence(x, new_seq)\n",
        "\n",
        "                num_grad = (loss_func(h, y) - loss) / epsilon # (f(x+eps) - f(x)) / eps\n",
        "                layer.db[b_i] = num_grad\n",
        "        \n",
        "        return loss\n",
        "\n",
        "\n",
        "def gradient_descent(network, x, y, loss_obj, alpha=0.01):\n",
        "    loss = network.calc_gradient(x, y, loss_obj)\n",
        "    for layer in network.sequence:\n",
        "        layer.W += -alpha * layer.dW\n",
        "        layer.b += -alpha * layer.db\n",
        "    return loss"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2NhRGf5EZJ5v",
        "outputId": "4ffcd240-52ab-451c-f3b0-10af816c9dbb"
      },
      "source": [
        "x = np.random.normal(0.0, 1.0, (10,))\n",
        "y = np.random.normal(0.0, 1.0, (2,))\n",
        "\n",
        "dnn = DNN(hidden_depth=10, num_neuron=32,num_input=10, num_output=2, activation=sigmoid)\n",
        "\n",
        "t = time.time()\n",
        "for epoch in range(100):\n",
        "    loss = gradient_descent(dnn, x, y, mean_squared_error, 0.01)\n",
        "    print('Epoch {}: Test loss{}'.format(epoch, loss))\n",
        "\n",
        "\n",
        "print('{} seconds elapsed.'.format(time.time() - t))\n",
        "\n",
        "# hidden_depth = 5: 48.27160286903381 seconds elapsed.\n",
        "# hidden_depth = 10: 154.6369285583496 seconds elapsed."
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0: Test loss0.06484947652243556\n",
            "Epoch 1: Test loss0.06448388702646624\n",
            "Epoch 2: Test loss0.06412043730242596\n",
            "Epoch 3: Test loss0.06375912569311103\n",
            "Epoch 4: Test loss0.06339995025618941\n",
            "Epoch 5: Test loss0.06304290876955439\n",
            "Epoch 6: Test loss0.06268799873667782\n",
            "Epoch 7: Test loss0.062335217391997086\n",
            "Epoch 8: Test loss0.06198456170626981\n",
            "Epoch 9: Test loss0.06163602839197021\n",
            "Epoch 10: Test loss0.061289613908666725\n",
            "Epoch 11: Test loss0.06094531446839861\n",
            "Epoch 12: Test loss0.0606031260410389\n",
            "Epoch 13: Test loss0.06026304435965804\n",
            "Epoch 14: Test loss0.05992506492586989\n",
            "Epoch 15: Test loss0.059589183015159315\n",
            "Epoch 16: Test loss0.05925539368218741\n",
            "Epoch 17: Test loss0.05892369176608198\n",
            "Epoch 18: Test loss0.05859407189569654\n",
            "Epoch 19: Test loss0.05826652849484582\n",
            "Epoch 20: Test loss0.057941055787510416\n",
            "Epoch 21: Test loss0.057617647802999744\n",
            "Epoch 22: Test loss0.05729629838109926\n",
            "Epoch 23: Test loss0.05697700117715808\n",
            "Epoch 24: Test loss0.05665974966715895\n",
            "Epoch 25: Test loss0.05634453715272932\n",
            "Epoch 26: Test loss0.05603135676611914\n",
            "Epoch 27: Test loss0.055720201475125054\n",
            "Epoch 28: Test loss0.055411064087984796\n",
            "Epoch 29: Test loss0.05510393725820435\n",
            "Epoch 30: Test loss0.05479881348933882\n",
            "Epoch 31: Test loss0.05449568513973648\n",
            "Epoch 32: Test loss0.05419454442721461\n",
            "Epoch 33: Test loss0.053895383433678896\n",
            "Epoch 34: Test loss0.053598194109707495\n",
            "Epoch 35: Test loss0.05330296827905657\n",
            "Epoch 36: Test loss0.05300969764312364\n",
            "Epoch 37: Test loss0.052718373785344004\n",
            "Epoch 38: Test loss0.052428988175527956\n",
            "Epoch 39: Test loss0.052141532174145896\n",
            "Epoch 40: Test loss0.051855997036547155\n",
            "Epoch 41: Test loss0.0515723739171117\n",
            "Epoch 42: Test loss0.05129065387335296\n",
            "Epoch 43: Test loss0.051010827869945076\n",
            "Epoch 44: Test loss0.05073288678270102\n",
            "Epoch 45: Test loss0.050456821402466136\n",
            "Epoch 46: Test loss0.05018262243897619\n",
            "Epoch 47: Test loss0.049910280524624935\n",
            "Epoch 48: Test loss0.0496397862181811\n",
            "Epoch 49: Test loss0.04937113000844558\n",
            "Epoch 50: Test loss0.049104302317826524\n",
            "Epoch 51: Test loss0.04883929350586977\n",
            "Epoch 52: Test loss0.048576093872708434\n",
            "Epoch 53: Test loss0.04831469366245823\n",
            "Epoch 54: Test loss0.04805508306654708\n",
            "Epoch 55: Test loss0.04779725222697648\n",
            "Epoch 56: Test loss0.04754119123951644\n",
            "Epoch 57: Test loss0.04728689015684805\n",
            "Epoch 58: Test loss0.04703433899163017\n",
            "Epoch 59: Test loss0.04678352771950985\n",
            "Epoch 60: Test loss0.04653444628206675\n",
            "Epoch 61: Test loss0.04628708458969602\n",
            "Epoch 62: Test loss0.04604143252442646\n",
            "Epoch 63: Test loss0.04579747994268152\n",
            "Epoch 64: Test loss0.045555216677977683\n",
            "Epoch 65: Test loss0.04531463254355566\n",
            "Epoch 66: Test loss0.04507571733495683\n",
            "Epoch 67: Test loss0.04483846083254453\n",
            "Epoch 68: Test loss0.04460285280395036\n",
            "Epoch 69: Test loss0.04436888300648312\n",
            "Epoch 70: Test loss0.04413654118945804\n",
            "Epoch 71: Test loss0.04390581709648511\n",
            "Epoch 72: Test loss0.0436767004676903\n",
            "Epoch 73: Test loss0.04344918104188676\n",
            "Epoch 74: Test loss0.04322324855868638\n",
            "Epoch 75: Test loss0.042998892760555636\n",
            "Epoch 76: Test loss0.04277610339482277\n",
            "Epoch 77: Test loss0.042554870215623944\n",
            "Epoch 78: Test loss0.04233518298580547\n",
            "Epoch 79: Test loss0.042117031478765686\n",
            "Epoch 80: Test loss0.04190040548024759\n",
            "Epoch 81: Test loss0.041685294790082796\n",
            "Epoch 82: Test loss0.04147168922388819\n",
            "Epoch 83: Test loss0.04125957861470272\n",
            "Epoch 84: Test loss0.04104895281458798\n",
            "Epoch 85: Test loss0.040839801696174904\n",
            "Epoch 86: Test loss0.04063211515416447\n",
            "Epoch 87: Test loss0.04042588310678315\n",
            "Epoch 88: Test loss0.04022109549718908\n",
            "Epoch 89: Test loss0.04001774229484147\n",
            "Epoch 90: Test loss0.03981581349681756\n",
            "Epoch 91: Test loss0.03961529912909103\n",
            "Epoch 92: Test loss0.03941618924776882\n",
            "Epoch 93: Test loss0.03921847394028553\n",
            "Epoch 94: Test loss0.039022143326554196\n",
            "Epoch 95: Test loss0.03882718756008072\n",
            "Epoch 96: Test loss0.03863359682903808\n",
            "Epoch 97: Test loss0.03844136135730088\n",
            "Epoch 98: Test loss0.0382504714054427\n",
            "Epoch 99: Test loss0.03806091727169576\n",
            "154.6369285583496 seconds elapsed.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0QXGEhAQ_XQ6"
      },
      "source": [
        "## 문제 2\n",
        "\n",
        "backprop_example_code.py 파일에는 \n",
        "\n",
        "역전파 알고리즘으로 구현한 DNN 예제 코드가 저장되어 있습니다.(from example_code import *)\n",
        "\n",
        "activation function을 **Tanh class**로 구현하여 \n",
        "\n",
        "DNN을 학습 시켜 보세요 (Tanh Class를 구현해 activation =Tanh 만 하면됩니다.)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wIo-ilF7_XQ7"
      },
      "source": [
        "## 역전파 학습법을 이용한 심층 신경망 학습\n",
        "\n",
        "# ## 유틸리티 함수\n",
        "import numpy as np\n",
        "import time \n",
        "\n",
        "def _t(x):\n",
        "    return np.transpose(x)\n",
        "\n",
        "def _m(A, B):\n",
        "    return np.matmul(A, B)\n",
        "\n",
        "class MeanSquaredError: # 1/2 * mean((h - y)^2)  --> h - y\n",
        "    def __init__(self):\n",
        "        self.dh = 1\n",
        "        self.last_diff = 1\n",
        "\n",
        "    def __call__(self, h, y):\n",
        "        self.last_diff = h - y\n",
        "        return 1 / 2 * np.mean(np.square(self.last_diff))\n",
        "\n",
        "    def grad(self):\n",
        "        return self.last_diff\n",
        "    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OedHCddB_XQ9",
        "scrolled": true
      },
      "source": [
        "# ## Activation Function\n",
        "\n",
        "# 1) sigmoid\n",
        "class Sigmoid:\n",
        "    def __init__(self):\n",
        "        self.last_o = 1\n",
        "\n",
        "    def __call__(self, x):\n",
        "        self.last_o = 1.0 / (1.0 + np.exp(-x))\n",
        "        return self.last_o\n",
        "\n",
        "    def grad(self): # sigmoid(x)(1 - sigmoid(x))\n",
        "        return self.last_o * (1.0 - self.last_o)\n",
        "\n",
        "\n",
        "# 2) Tanh\n",
        "\n",
        "class Tanh:\n",
        "    def __init__(self):\n",
        "        self.last_o = 1\n",
        "\n",
        "    def __call__(self, x):\n",
        "        self.last_o = (np.exp(x) - np.exp(-x))/(np.exp(x) + np.exp(-x))\n",
        "        return self.last_o\n",
        "\n",
        "    def grad(self): # (1 + tanh(x))(1 - tanh(x))\n",
        "        return (1 + self.last_o) * (1 - self.last_o)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J8f9W6RnZJ52"
      },
      "source": [
        "# Dense 클래스 구현\n",
        "# (weight, bias)\n",
        "# 역전파까지 함께 정의\n",
        "class Dense:\n",
        "    def __init__(self, W, b, a_obj):\n",
        "        self.W = W\n",
        "        self.b = b\n",
        "        self.a = a_obj()\n",
        "        \n",
        "        self.dW = np.zeros_like(self.W)\n",
        "        self.db = np.zeros_like(self.b)\n",
        "        self.dh = np.zeros_like(_t(self.W))\n",
        "        \n",
        "        self.last_x = np.zeros((self.W.shape[0]))\n",
        "        self.last_h = np.zeros((self.W.shape[1]))\n",
        "        \n",
        "\n",
        "    def __call__(self, x):\n",
        "        self.last_x = x\n",
        "        self.last_h = _m(_t(self.W), x) + self.b\n",
        "        return self.a(self.last_h)\n",
        "\n",
        "    def grad(self): # dy/dh = W\n",
        "        return self.W * self.a.grad()\n",
        "\n",
        "    def grad_W(self, dh):\n",
        "        grad = np.ones_like(self.W)\n",
        "        grad_a = self.a.grad()\n",
        "        for j in range(grad.shape[1]): # dy/dw = x\n",
        "            grad[:, j] = dh[j] * grad_a[j] * self.last_x\n",
        "        return grad\n",
        "\n",
        "    def grad_b(self, dh): # dy/db = 1\n",
        "        return dh * self.a.grad()\n",
        "\n",
        "\n",
        "# 심층 신경망 클래스 구현\n",
        "    # layer 개수만큼 정의\n",
        "class DNN:\n",
        "    def __init__(self, hidden_depth, num_neuron, input, output, activation=Sigmoid):\n",
        "        def init_var(i, o):\n",
        "            return np.random.normal(0.0, 0.01, (i, o)), np.zeros((o,))\n",
        "\n",
        "        self.sequence = list()\n",
        "        # First hidden layer\n",
        "        W, b = init_var(input, num_neuron)\n",
        "        self.sequence.append(Dense(W, b, activation))\n",
        "\n",
        "        # Hidden Layers\n",
        "        for index in range(hidden_depth):\n",
        "            W, b = init_var(num_neuron, num_neuron)\n",
        "            self.sequence.append(Dense(W, b, activation))\n",
        "\n",
        "        # Output Layer\n",
        "        W, b = init_var(num_neuron, output)\n",
        "        self.sequence.append(Dense(W, b, activation))\n",
        "\n",
        "    def __call__(self, x):\n",
        "        for layer in self.sequence:\n",
        "            x = layer(x)\n",
        "        return x\n",
        "\n",
        "    def calc_gradient(self, loss_obj):\n",
        "        loss_obj.dh = loss_obj.grad()\n",
        "        self.sequence.append(loss_obj)\n",
        "        \n",
        "        # back-prop loop\n",
        "        for i in range(len(self.sequence) - 1, 0, -1):\n",
        "            l1 = self.sequence[i]\n",
        "            l0 = self.sequence[i - 1]\n",
        "            \n",
        "            l0.dh = _m(l0.grad(), l1.dh)\n",
        "            l0.dW = l0.grad_W(l1.dh)\n",
        "            l0.db = l0.grad_b(l1.dh)\n",
        "        \n",
        "        self.sequence.remove(loss_obj)\n",
        "\n",
        "\n",
        "# ## 경사하강 학습법\n",
        "\n",
        "def gradient_descent(network, x, y, loss_obj, alpha=0.01):\n",
        "    loss = loss_obj(network(x), y)  # Forward inference\n",
        "    network.calc_gradient(loss_obj)  # Back-propagation\n",
        "    for layer in network.sequence:\n",
        "        layer.W += -alpha * layer.dW\n",
        "        layer.b += -alpha * layer.db\n",
        "    return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l1sbNuT8ZJ5_",
        "outputId": "d41ca71b-7d6e-4241-b6fc-5751f5234179"
      },
      "source": [
        "x = np.random.normal(0.0, 1.0, (10,))\n",
        "y = np.random.normal(0.0, 1.0, (2,))\n",
        "\n",
        "dnn = DNN(hidden_depth=5, num_neuron=32, input=10, output=2, activation=Tanh)\n",
        "\n",
        "t = time.time()\n",
        "for epoch in range(100):\n",
        "    loss = MeanSquaredError()\n",
        "    print('Epoch {}: Test loss{}'.format(epoch, loss))\n",
        "\n",
        "print('{} seconds elapsed.'.format(time.time() - t))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0: Test loss<__main__.MeanSquaredError object at 0x0000028D57919CF8>\n",
            "Epoch 1: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 2: Test loss<__main__.MeanSquaredError object at 0x0000028D57919CF8>\n",
            "Epoch 3: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 4: Test loss<__main__.MeanSquaredError object at 0x0000028D57919CF8>\n",
            "Epoch 5: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 6: Test loss<__main__.MeanSquaredError object at 0x0000028D57919CF8>\n",
            "Epoch 7: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 8: Test loss<__main__.MeanSquaredError object at 0x0000028D57919CF8>\n",
            "Epoch 9: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 10: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 11: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 12: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 13: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 14: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 15: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 16: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 17: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 18: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 19: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 20: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 21: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 22: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 23: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 24: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 25: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 26: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 27: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 28: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 29: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 30: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 31: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 32: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 33: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 34: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 35: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 36: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 37: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 38: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 39: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 40: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 41: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 42: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 43: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 44: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 45: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 46: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 47: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 48: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 49: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 50: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 51: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 52: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 53: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 54: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 55: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 56: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 57: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 58: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 59: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 60: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 61: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 62: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 63: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 64: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 65: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 66: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 67: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 68: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 69: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 70: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 71: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 72: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 73: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 74: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 75: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 76: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 77: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 78: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 79: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 80: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 81: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 82: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 83: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 84: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 85: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 86: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 87: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 88: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 89: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 90: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 91: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 92: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 93: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 94: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 95: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 96: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 97: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "Epoch 98: Test loss<__main__.MeanSquaredError object at 0x0000028D57D440F0>\n",
            "Epoch 99: Test loss<__main__.MeanSquaredError object at 0x0000028D57D0B438>\n",
            "0.01578831672668457 seconds elapsed.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KHoZEJ1P_XQ9"
      },
      "source": [
        "### 문제 3 \n",
        "\n",
        "두가지 방식으로 구현한 DNN (수치적 미분/ 역전파 알고리즘)에는 확인한 바와 같이 \n",
        "\n",
        "학습 속도가 명확히 차이가 나는 것을 확인 할 수 있습니다.\n",
        "\n",
        "역전파 알고리즘이 학습이 빠른 이유를 설명하고, 어떠한 방식으로 구현하고 있나요?:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gNjbGRCoZJ6C"
      },
      "source": [
        "\"\"\"\r\n",
        "수치적 미분은 각 스칼라 변수를 조금씩 바꾸어 대입해 보면서 수치적 기울기를 구한다.  \r\n",
        "그래서 N개의 매개변수를 가지고 있다면, 이를 미분하기 위해서 N+1번 더 손실함수를 평가해야 한다.\r\n",
        "이는 경사하강법 한 스텝 계산을 위해 N(N+1)의 곱하기 연산을 해야함을 뜻한다.\r\n",
        "하지만 역전파 알고리즘은 정방향 연산을 하여 loss를 구하는 과정에서 중간 결과를 저장하고, 이후 loss를 각 파라미터로 미분하는 과정에서 직렬 연결된 두 함수의 미분을 위해 연쇄 법칙(역방향 연산)을 이용한다.\r\n",
        "이에 따라 역전파 알고리즘 학습은 단 한번의 손실함수 평가로 미분을 구하게 되면서 훨씬 빠른 속도로 학습이 된다.\r\n",
        "\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vHlFHZ2A_XQ-"
      },
      "source": [
        "### 문제 4 _뉴럴 네트워크 학습 알고리즘 구현\n",
        "  \n",
        "1. tensorflow를 임포트 하고 하이퍼파라미터 epoch를 10으로 정의하세요."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q30ySHsf_XQ-"
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "EPOCHS = 10"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_y8GHiDd_XQ_"
      },
      "source": [
        "2. __init__, call 함수를 사용하여 네트워크 구조 정의"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7eniySvA_XRA"
      },
      "source": [
        "class Mymodel(tf.keras.Model):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.flatten = tf.keras.layers.Flatten(input_shape = (28,28))\n",
        "        self.dense1 = tf.keras.layers.Dense(32, activation = 'relu')\n",
        "        self.dense2 = tf.keras.layers.Dense(64, activation = 'relu')\n",
        "        self.dense3 = tf.keras.layers.Dense(128, activation = 'relu')\n",
        "        self.dense4 = tf.keras.layers.Dense(256, activation = 'relu')\n",
        "        self.dense5 = tf.keras.layers.Dense(10, activation = 'softmax')\n",
        "    def call(self, x, traning = None, mask = None):\n",
        "        x = self.flatten(x)\n",
        "        x = self.dense1(x)\n",
        "        x = self.dense2(x)\n",
        "        x = self.dense3(x)\n",
        "        x = self.dense4(x)\n",
        "        return self.dense5(x)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XuQbiL59_XRB"
      },
      "source": [
        "3. 학습함수와 테스트 함수를 구현하세요."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5slVCbHQ_XRB"
      },
      "source": [
        "# 학습함수 구현\n",
        "\n",
        "@tf.function\n",
        "def train_step(model, images, labels, loss_object, optimizer, train_loss, train_accuracy):\n",
        "    with tf.GradientTape() as tape:\n",
        "        predictions = model(images)\n",
        "        loss =loss_object(labels, predictions)\n",
        "        \n",
        "    gradients = tape.gradient(loss, model.trainable_variables)\n",
        "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
        "    \n",
        "    train_loss(loss)\n",
        "    train_accuracy(labels, predictions)\n",
        "    \n",
        "    \n",
        "# 테스트 함수 구현\n",
        "@tf.function\n",
        "def test_step(model, images, labels, loss_object, test_loss, test_accuracy):\n",
        "    predictions = model(images)\n",
        "    loss = loss_object(labels, predictions)\n",
        "    \n",
        "    test_loss(loss)\n",
        "    test_accuracy(labels, predictions)\n",
        "             \n"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M7dctXB3_XRC"
      },
      "source": [
        "4. 복습과제에 함께 첨부한 SVHN 데이터를 불러오세요.\n",
        "  - scipy.io를 임포트하여 데이터 불러오기\n",
        "  - 'train_32x32.mat' / 'test_32x32.mat'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1G8KMCBhZfJl",
        "outputId": "04efc4f1-53ec-4d39-9842-8a892696a4d8"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r0bh7GkF_XRD"
      },
      "source": [
        "import numpy as np\n",
        "import scipy.io as sio\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "path = '/content/drive/MyDrive/Colab Notebooks/Bitamin_머신러닝 심화/2-15 딥러닝-3 복습'\n",
        "\n",
        "train_data = sio.loadmat(path + '/train_32x32.mat')\n",
        "test_data = sio.loadmat(path +'/test_32x32.mat')\n",
        "\n",
        "\n",
        "x_train = np.array(train_data['X'])\n",
        "x_test = np.array(test_data['X'])\n",
        "\n",
        "y_train = train_data['y']\n",
        "y_test = test_data['y']"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fILeA7py_XRD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "661866a7-1632-4f79-d421-2bdd9b8a55bf"
      },
      "source": [
        "# Fix the axes of the images\n",
        "\n",
        "x_train = np.moveaxis(x_train, -1, 0)\n",
        "x_test = np.moveaxis(x_test, -1, 0)\n",
        "\n",
        "print(x_train.shape)\n",
        "print(x_test.shape)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(73257, 32, 32, 3)\n",
            "(26032, 32, 32, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aUFDaAs7_XRF"
      },
      "source": [
        "* 파라미터는 아래와 같습니다. \n",
        "  - shuffle = 1024, batch = 32\n",
        "* from_tensor_slices 할 때 라벨 범위 조정을 위해 y는 -1한 숫자를 넣습니다. (ex. y_train -> y_train-1)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VrnXDl3Q_XRG"
      },
      "source": [
        "train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train-1)).shuffle(1024).batch(32)\n",
        "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test-1)).batch(32)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uy23CrV7_XRG"
      },
      "source": [
        "5. 모델을 생성하고 손실함수와 최적화 알고리즘을 정의하세요\n",
        "6. 성능지표를 정의하세요"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kfsRr1CY_XRH"
      },
      "source": [
        "#  모델 생성\r\n",
        "\r\n",
        "model = Mymodel()\r\n",
        "\r\n",
        "# 손실 함수 및 최적화 알고리즘 정의\r\n",
        "\r\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\r\n",
        "optimizer = tf.keras.optimizers.Adam()"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jyMt0yD7_XRH"
      },
      "source": [
        "# 성능 지표 정의\r\n",
        "\r\n",
        "train_loss = tf.keras.metrics.Mean(name = 'train_loss')\r\n",
        "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name = 'train_accuracy')\r\n",
        "\r\n",
        "test_loss = tf.keras.metrics.Mean(name = 'test_loss')\r\n",
        "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name = 'test_accuracy')"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5HgRI5fd_XRH"
      },
      "source": [
        "7. for문을 사용하여 실제 데이터 함수에 적용하세요. (학습 루프 구현)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qxXEYjDQ_XRI",
        "scrolled": false,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a6a8290-5e1a-44f8-f6db-8282c2284aa5"
      },
      "source": [
        "for epoch in range(EPOCHS):\r\n",
        "    for images, labels in train_ds:\r\n",
        "        train_step(model, images, labels, loss_object, optimizer, train_loss, train_accuracy)\r\n",
        "    for images, labels in test_ds:\r\n",
        "        test_step(model, images, labels, loss_object, test_loss, test_accuracy)\r\n",
        "        \r\n",
        "    template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\r\n",
        "    print(template.format(epoch+1,\r\n",
        "                         train_loss.result(),\r\n",
        "                         train_accuracy.result() *100,\r\n",
        "                         test_loss.result(),\r\n",
        "                         test_accuracy.result() * 100))\r\n",
        "\r\n",
        "    train_loss.reset_states()\r\n",
        "    train_accuracy.reset_states()\r\n",
        "    test_loss.reset_states()\r\n",
        "    test_accuracy.reset_states()\r\n"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1, Loss: 2.5203797817230225, Accuracy: 18.792743682861328, Test Loss: 2.22653865814209, Test Accuracy: 19.587430953979492\n",
            "Epoch 2, Loss: 2.2374134063720703, Accuracy: 18.921058654785156, Test Loss: 2.227358102798462, Test Accuracy: 19.587430953979492\n",
            "Epoch 3, Loss: 2.237299680709839, Accuracy: 18.921058654785156, Test Loss: 2.2277166843414307, Test Accuracy: 19.587430953979492\n",
            "Epoch 4, Loss: 2.23742938041687, Accuracy: 18.921058654785156, Test Loss: 2.227597236633301, Test Accuracy: 19.587430953979492\n",
            "Epoch 5, Loss: 2.2373063564300537, Accuracy: 18.921058654785156, Test Loss: 2.2279269695281982, Test Accuracy: 19.587430953979492\n",
            "Epoch 6, Loss: 2.237198829650879, Accuracy: 18.921058654785156, Test Loss: 2.2267038822174072, Test Accuracy: 19.587430953979492\n",
            "Epoch 7, Loss: 2.237062454223633, Accuracy: 18.921058654785156, Test Loss: 2.226311206817627, Test Accuracy: 19.587430953979492\n",
            "Epoch 8, Loss: 2.237055540084839, Accuracy: 18.921058654785156, Test Loss: 2.226062774658203, Test Accuracy: 19.587430953979492\n",
            "Epoch 9, Loss: 2.2370028495788574, Accuracy: 18.921058654785156, Test Loss: 2.226031541824341, Test Accuracy: 19.587430953979492\n",
            "Epoch 10, Loss: 2.236997365951538, Accuracy: 18.921058654785156, Test Loss: 2.2256102561950684, Test Accuracy: 19.587430953979492\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EnOyXNEX_XRI"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8jVoRe3N_XRJ"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}