lec_4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fe6e5d71",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from pathlib import Path\n",
    "import gzip, pickle\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c16d885",
   "metadata": {},
   "source": [
    "#### load  data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e1d5b33d",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_FOLDER = Path('data')\n",
    "DATA_FILE = DATA_FOLDER/'mnist.pkl.gz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d8ab35b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open(DATA_FILE, 'rb') as f:\n",
    "    data = pickle.load(f, encoding='latin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fd30721b",
   "metadata": {},
   "outputs": [],
   "source": [
    "((x_train, y_train), (x_val, y_val), _) = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8bcf6aac",
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train, x_val, y_val) = map(torch.tensor, (x_train, y_train, x_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e8702d48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([50000, 784]),\n",
       " torch.Size([50000]),\n",
       " torch.Size([10000, 784]),\n",
       " torch.Size([10000]))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape, y_train.shape, x_val.shape, y_val.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43c55937",
   "metadata": {},
   "source": [
    "#### model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "ff6e8647",
   "metadata": {},
   "outputs": [],
   "source": [
    "nin, nh, nout = x_train.shape[1], 50, 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "c2540620",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self, nin, nh, nout):\n",
    "        super().__init__()\n",
    "        self.layers = [nn.Linear(nin, nh), nn.ReLU(nh), nn.Linear(nh, nout)]\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        for l in self.layers:\n",
    "            x = l(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "72c8bd16",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(nin, nh, nout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "11793af2",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = model(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "b1395d22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50000, 10])"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "f098412a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 2.3653e-01,  5.7953e-04, -1.8526e-01,  ..., -1.1290e-03,\n",
       "          1.0068e-01, -1.2474e-01],\n",
       "        [ 2.2510e-01, -9.3672e-02, -2.3261e-01,  ...,  5.9222e-02,\n",
       "          1.9832e-02, -8.4995e-02],\n",
       "        [ 1.9867e-01, -1.5230e-01, -6.5086e-02,  ...,  1.8481e-02,\n",
       "         -1.1236e-02, -1.4206e-01],\n",
       "        ...,\n",
       "        [ 2.4863e-01, -2.2664e-01, -9.7118e-02,  ..., -1.3454e-01,\n",
       "          1.5280e-01, -1.7664e-01],\n",
       "        [ 2.1275e-01, -1.0548e-01, -3.1598e-02,  ..., -3.2585e-02,\n",
       "          9.3807e-02, -1.0054e-01],\n",
       "        [ 1.9768e-01, -1.7176e-01, -1.0908e-01,  ...,  2.7064e-05,\n",
       "          3.1564e-02, -8.3367e-02]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "1c902ea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_softmax(x):\n",
    "    return (x.exp()/ x.exp().sum(-1, keepdim=True)).log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "3aea94d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-2.1122, -2.3481, -2.5340,  ..., -2.3498, -2.2480, -2.4735],\n",
       "        [-2.1032, -2.4220, -2.5609,  ..., -2.2691, -2.3085, -2.4133],\n",
       "        [-2.1102, -2.4612, -2.3740,  ..., -2.2904, -2.3201, -2.4509],\n",
       "        ...,\n",
       "        [-2.0636, -2.5389, -2.4094,  ..., -2.4468, -2.1595, -2.4889],\n",
       "        [-2.1168, -2.4350, -2.3611,  ..., -2.3621, -2.2357, -2.4301],\n",
       "        [-2.1254, -2.4949, -2.4322,  ..., -2.3231, -2.2916, -2.4065]],\n",
       "       grad_fn=<LogBackward0>)"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_softmax(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "bd4cc9c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def logsumexp(x):\n",
    "    m = x.max(-1)[0]\n",
    "    return m + (x-m[:,None]).exp().sum(-1).log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "87838cbf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2.3487, 2.3283, 2.3089,  ..., 2.3123, 2.3295, 2.3231],\n",
       "       grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logsumexp(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "b08c1dba",
   "metadata": {},
   "outputs": [],
   "source": [
    "def logsumexp_pyt(x):\n",
    "    return x - x.logsumexp(-1, keepdim=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "66e509aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-2.1122, -2.3481, -2.5340,  ..., -2.3498, -2.2480, -2.4735],\n",
       "        [-2.1032, -2.4220, -2.5609,  ..., -2.2691, -2.3085, -2.4133],\n",
       "        [-2.1102, -2.4612, -2.3740,  ..., -2.2904, -2.3201, -2.4509],\n",
       "        ...,\n",
       "        [-2.0636, -2.5389, -2.4094,  ..., -2.4468, -2.1595, -2.4889],\n",
       "        [-2.1168, -2.4350, -2.3611,  ..., -2.3621, -2.2357, -2.4301],\n",
       "        [-2.1254, -2.4949, -2.4322,  ..., -2.3231, -2.2916, -2.4065]],\n",
       "       grad_fn=<SubBackward0>)"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logsumexp_pyt(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "5f6dfa8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.allclose(log_softmax(pred), logsumexp_pyt(pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "612953e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5, 0, 4])"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "60af16ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "sm_pred = logsumexp_pyt(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "a4bb4807",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50000, 10])"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "055a51fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(-2.2889, grad_fn=<SelectBackward0>),\n",
       " tensor(-2.1032, grad_fn=<SelectBackward0>),\n",
       " tensor(-2.1724, grad_fn=<SelectBackward0>))"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_pred[0, 5], sm_pred[1, 0], sm_pred[2, 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "253cadec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-2.2889, -2.1032, -2.1724], grad_fn=<IndexBackward0>)"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sm_pred[[0,1,2], y_train[:3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "27269098",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nll(input, target):\n",
    "    return - input[range(target.shape[0]), target].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "0904a692",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.3053, grad_fn=<NegBackward0>)"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = nll(sm_pred, y_train)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "617c2b73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50000, 10])"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "163f7156",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-2.1122, -2.3481, -2.5340,  ..., -2.3498, -2.2480, -2.4735],\n",
       "        [-2.1032, -2.4220, -2.5609,  ..., -2.2691, -2.3085, -2.4133],\n",
       "        [-2.1102, -2.4612, -2.3740,  ..., -2.2904, -2.3201, -2.4509],\n",
       "        ...,\n",
       "        [-2.0636, -2.5389, -2.4094,  ..., -2.4468, -2.1595, -2.4889],\n",
       "        [-2.1168, -2.4350, -2.3611,  ..., -2.3621, -2.2357, -2.4301],\n",
       "        [-2.1254, -2.4949, -2.4322,  ..., -2.3231, -2.2916, -2.4065]],\n",
       "       grad_fn=<LogSoftmaxBackward0>)"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.log_softmax(pred, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "aa7f63d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.allclose(F.nll_loss(F.log_softmax(pred, -1), y_train), loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "ab10b493",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.allclose(F.cross_entropy(pred, y_train), loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "00aa78e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "e409f531",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50, 784])"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xb = x_train[0:bs]\n",
    "xb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "d202af70",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50])"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yb = y_train[0:bs]\n",
    "yb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "890efec3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5, 0, 4, 1, 9, 2, 1, 3, 1, 4, 3, 5, 3, 6, 1, 7, 2, 8, 6, 9, 4, 0, 9, 1,\n",
       "        1, 2, 4, 3, 2, 7, 3, 8, 6, 9, 0, 5, 6, 0, 7, 6, 1, 8, 7, 9, 3, 9, 8, 5,\n",
       "        9, 3])"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "ef17165d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = model(xb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "ffb89216",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50, 10])"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "8dcd8e1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0800)"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "699498a7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45d7d363",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "9ce54ec9",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.5\n",
    "epoches = 5\n",
    "n = x_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "4669aeef",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_loss = F.cross_entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "7115caf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(pred, target):\n",
    "    return (pred.argmax(1) == target).float().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b9da257e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def report(loss, pred, target):\n",
    "    return print(f'loss:{loss:.2f}, accuracy:{accuracy(pred, target):.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "id": "dc40019f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:2.33, accuracy:0.02\n"
     ]
    }
   ],
   "source": [
    "xb,yb = x_train[0:bs], y_train[0:bs]\n",
    "model = Model(nin, nh, nout)\n",
    "pred = model(xb)\n",
    "loss = get_loss(pred, yb)\n",
    "report(loss, pred, yb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "3fcb608f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.14, accuracy:0.94\n",
      "loss:0.10, accuracy:1.00\n",
      "loss:0.05, accuracy:1.00\n",
      "loss:0.05, accuracy:0.98\n",
      "loss:0.04, accuracy:1.00\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epoches):\n",
    "    for i in range(0, n, bs):\n",
    "        s = slice(i, min(n, i+bs))\n",
    "        xb,yb = x_train[s], y_train[s]\n",
    "        pred = model(xb)\n",
    "        loss = get_loss(pred, yb)\n",
    "        loss.backward()\n",
    "        with torch.no_grad():\n",
    "            for l in model.layers:\n",
    "                if hasattr(l, 'weight'):\n",
    "                    l.weight -= lr * l.weight.grad\n",
    "                    l.bias -= lr * l.bias.grad\n",
    "                    l.weight.grad.zero_()\n",
    "                    l.bias.grad.zero_()\n",
    "    report(loss, pred, yb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "2e491ac5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5, 2, 0, 0, 0, 3, 9, 5, 7, 6, 6, 1, 2, 8, 1, 6, 3, 4, 3, 6, 6, 0, 1, 9,\n",
       "        0, 2, 2, 6, 7, 4, 7, 3, 4, 9, 9, 3, 7, 0, 9, 0, 8, 5, 5, 2, 4, 5, 0, 8,\n",
       "        4, 8])"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred.argmax(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "867bd9d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([5, 2, 0, 0, 0, 3, 9, 5, 7, 6, 6, 1, 2, 8, 1, 6, 3, 4, 3, 6, 6, 0, 1, 9,\n",
       "        0, 2, 2, 6, 7, 4, 7, 3, 4, 9, 9, 3, 7, 0, 9, 0, 8, 5, 5, 2, 4, 5, 0, 8,\n",
       "        4, 8])"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "1dad731d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "517d8569",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = [1, 2, 3, 5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "ef16beab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduce(lambda val, op: val + op, t, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31b737ad",
   "metadata": {},
   "source": [
    "#### params and optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b1456846",
   "metadata": {},
   "outputs": [],
   "source": [
    "m1 = nn.Module()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "203bfd0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "m1.foo = nn.Linear(3, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "014d5e05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Module(\n",
       "  (foo): Linear(in_features=3, out_features=4, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1fd019a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('foo', Linear(in_features=3, out_features=4, bias=True))]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(m1.named_children())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d3208a1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.nn.modules.module.Module'>\n"
     ]
    }
   ],
   "source": [
    "print(nn.Module)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9c7b7bb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.Module??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7623d4dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self, nin, nh, nout):\n",
    "        super().__init__()\n",
    "        self.l1 = nn.Linear(nin, nh)\n",
    "        self.l2 = nn.Linear(nh, nout)\n",
    "        self.relu = nn.ReLU()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.l2(self.relu(self.l1(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "b7d1faca",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = x_train.shape[1]\n",
    "m = x_train.shape[0]\n",
    "nh = 50\n",
    "no = 10\n",
    "epoches = 3\n",
    "lr = 0.5\n",
    "bs = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c714e66c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MLP(n, nh, no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ff2b119c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLP(\n",
       "  (l1): Linear(in_features=784, out_features=50, bias=True)\n",
       "  (l2): Linear(in_features=50, out_features=10, bias=True)\n",
       "  (relu): ReLU()\n",
       ")"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "69b8adf7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: l1, l: Linear(in_features=784, out_features=50, bias=True)\n",
      "name: l2, l: Linear(in_features=50, out_features=10, bias=True)\n",
      "name: relu, l: ReLU()\n"
     ]
    }
   ],
   "source": [
    "for name, l in model.named_children():\n",
    "    print(f\"name: {name}, l: {l}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "04a58d2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([50, 784])\n",
      "torch.Size([50])\n",
      "torch.Size([10, 50])\n",
      "torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "for p in model.parameters():\n",
    "    print(p.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "6e45698a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit():\n",
    "    for epoch in range(epoches):\n",
    "        for i in range(0, m, bs):\n",
    "            s = slice(i, min(m, i+bs))\n",
    "            xb, yb = x_train[s], y_train[s]\n",
    "            pred = model(xb)\n",
    "            loss = F.cross_entropy(pred, yb)\n",
    "            loss.backward()\n",
    "            with torch.no_grad():\n",
    "                for p in model.parameters():\n",
    "                    p -= lr * p.grad\n",
    "                model.zero_grad()\n",
    "        \n",
    "        report(loss, pred, yb)              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "33d8e650",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.12, accuracy:0.94\n",
      "loss:0.10, accuracy:0.94\n",
      "loss:0.08, accuracy:0.96\n"
     ]
    }
   ],
   "source": [
    "fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "2b44b0ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyModule():\n",
    "    def __init__(self, nin, nh, nout):\n",
    "        self._modules = {}\n",
    "        self.l1 = nn.Linear(nin, nh)\n",
    "        self.l2 = nn.Linear(nh, nout)\n",
    "    \n",
    "    def __setattr__(self, k, v):\n",
    "        if not k.startswith('_'):\n",
    "            self._modules[k] = v\n",
    "        super().__setattr__(k, v)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return (f'{self._modules}')\n",
    "    \n",
    "    def parameters(self):\n",
    "        for m in self._modules.values():\n",
    "            yield from m.parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "92762d74",
   "metadata": {},
   "outputs": [],
   "source": [
    "m1 = MyModule(n, nh, no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "6dbc820c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'l1': Linear(in_features=784, out_features=50, bias=True), 'l2': Linear(in_features=50, out_features=10, bias=True)}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "dfcd35d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([50, 784])\n",
      "torch.Size([50])\n",
      "torch.Size([10, 50])\n",
      "torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "for p in m1.parameters():\n",
    "    print(p.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e4c2982",
   "metadata": {},
   "source": [
    "#### registering modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "3717caa1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "bab4129c",
   "metadata": {},
   "outputs": [],
   "source": [
    "layers = [nn.Linear(n, nh), nn.ReLU(), nn.Linear(nh, no)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "32745c04",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    def __init__(self, layers):\n",
    "        super().__init__()\n",
    "        self.layers = layers\n",
    "        for i, l in enumerate(layers):\n",
    "            self.add_module(f'layer_{i}', l)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return reduce(lambda x, l: l(x), self.layers, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "c7959b80",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "c371b12f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Model(\n",
       "  (layer_0): Linear(in_features=784, out_features=50, bias=True)\n",
       "  (layer_1): ReLU()\n",
       "  (layer_2): Linear(in_features=50, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "14f1f4ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50000, 10])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(x_train).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30200109",
   "metadata": {},
   "source": [
    "#### nn.ModuleList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "2d321990",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SequentialModel(nn.Module):\n",
    "    def __init__(self, layers):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList(layers)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        for l in self.layers:\n",
    "            x = l(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "3d1c2a58",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SequentialModel(layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "4904806c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequentialModel(\n",
       "  (layers): ModuleList(\n",
       "    (0): Linear(in_features=784, out_features=50, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Linear(in_features=50, out_features=10, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "d82f7683",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.14, accuracy:0.96\n",
      "loss:0.09, accuracy:0.96\n",
      "loss:0.07, accuracy:0.98\n"
     ]
    }
   ],
   "source": [
    "fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1bce7fe",
   "metadata": {},
   "source": [
    "#### nn.Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "6f441bc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = nn.Sequential(*layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "9c04e1e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=784, out_features=50, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=50, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "9e845bf4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.06, accuracy:0.98\n",
      "loss:0.07, accuracy:0.98\n",
      "loss:0.05, accuracy:1.00\n"
     ]
    }
   ],
   "source": [
    "fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2db3400",
   "metadata": {},
   "source": [
    "#### optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "804a7f28",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Optimizer():\n",
    "    def __init__(self, params, lr):\n",
    "        self.params = list(params)\n",
    "        self.lr = lr\n",
    "    \n",
    "    def step(self):\n",
    "        with torch.no_grad():\n",
    "            for p in self.params:\n",
    "                p -= p.grad * lr\n",
    "    \n",
    "    def zero_grad(self):\n",
    "        for p in self.params:\n",
    "            p.grad.data.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "3981b77e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = nn.Sequential(*layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "8cc22a9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=784, out_features=50, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=50, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "78aefbb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "optim = Optimizer(model.parameters(), lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "9c476137",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.12, accuracy:0.96\n",
      "loss:0.07, accuracy:1.00\n",
      "loss:0.09, accuracy:0.96\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epoches):\n",
    "    for i in range(0, m, bs):\n",
    "        s = slice(i, min(m, i+bs))\n",
    "        xb, yb = x_train[s], y_train[s]\n",
    "        pred = model(xb)\n",
    "        loss = F.cross_entropy(pred, yb)\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "        optim.zero_grad()\n",
    "        \n",
    "    report(loss, pred, yb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da776c86",
   "metadata": {},
   "source": [
    "#### optim.SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "3be2c471",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "0c7a66b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    layers = [nn.Linear(n, nh), nn.ReLU(), nn.Linear(nh, no)]\n",
    "    model = nn.Sequential(*layers)\n",
    "    opt = optim.SGD(model.parameters(), lr) \n",
    "    return model, opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "6619e45a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model, opt = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "e2659e3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=784, out_features=50, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=50, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "5e3107db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.17, accuracy:0.92\n",
      "loss:0.19, accuracy:0.94\n",
      "loss:0.12, accuracy:0.96\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epoches):\n",
    "    for i in range(0, m, bs):\n",
    "        s = slice(i, min(m, i+bs))\n",
    "        xb, yb = x_train[s], y_train[s]\n",
    "        pred = model(xb)\n",
    "        loss = F.cross_entropy(pred, yb)\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "        \n",
    "    report(loss, pred, yb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a33a502",
   "metadata": {},
   "source": [
    "#### datasets and dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "67bea032",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dataset():\n",
    "    def __init__(self, x, y):\n",
    "        self.x = x\n",
    "        self.y = y\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.x)\n",
    "    \n",
    "    def __getitem__(self, i):\n",
    "        return self.x[i], self.y[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "c53553aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds, val_ds = Dataset(x_train, y_train), Dataset(x_val, y_val) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "d5445942",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(x_train) == len(train_ds)\n",
    "assert len(x_val) == len(val_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "0d830288",
   "metadata": {},
   "outputs": [],
   "source": [
    "xb, yb = train_ds[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "5cada907",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 784])"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "e242f282",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5])"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "b806aa28",
   "metadata": {},
   "outputs": [],
   "source": [
    "model, opt = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "c36ce281",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.17, accuracy:0.94\n",
      "loss:0.12, accuracy:0.96\n",
      "loss:0.11, accuracy:0.96\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epoches):\n",
    "    for i in range(0, m, bs):\n",
    "        s = slice(i, min(m, i+bs))\n",
    "        xb, yb = train_ds[s]\n",
    "        pred = model(xb)\n",
    "        loss = F.cross_entropy(pred, yb)\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "    \n",
    "    report(loss, pred, yb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46ef4c64",
   "metadata": {},
   "source": [
    "#### dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "a95ec51b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataLoader():\n",
    "    def __init__(self, ds, bs):\n",
    "        self.ds = ds\n",
    "        self.bs = bs\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for i in range(0, len(self.ds), bs):\n",
    "            yield self.ds[i: i+self.bs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "35c8c7e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(train_ds, bs)\n",
    "val_dl = DataLoader(val_ds, bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "d1ea701d",
   "metadata": {},
   "outputs": [],
   "source": [
    "xb, yb = next(iter(train_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "bebb0dd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "f3a885aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1ab146e10>"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOX0lEQVR4nO3dbYxc5XnG8euKbUwxJvHGseMQFxzjFAg0Jl0ZkBFQoVCCIgGKCLGiiFBapwlOQutKUFoVWtHKrRIiSimSKS6m4iWQgPAHmsSyECRqcFmoAROHN+MS4+0aswIDIfZ6fffDjqsFdp5dZs68eO//T1rNzLnnzLk1cPmcmeeceRwRAjD5faDTDQBoD8IOJEHYgSQIO5AEYQeSmNrOjR3i6XGoZrRzk0Aqv9Fb2ht7PFatqbDbPkfS9ZKmSPrXiFhVev6hmqGTfVYzmwRQsDE21K01fBhve4qkGyV9TtLxkpbZPr7R1wPQWs18Zl8i6fmI2BoReyXdJem8atoCULVmwn6kpF+Nery9tuwdbC+33We7b0h7mtgcgGY0E/axvgR4z7m3EbE6InojoneapjexOQDNaCbs2yXNH/X445J2NNcOgFZpJuyPSlpke4HtQyR9SdK6atoCULWGh94iYp/tFZJ+rJGhtzUR8XRlnQGoVFPj7BHxgKQHKuoFQAtxuiyQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJNDWLK7qfp5b/E0/5yOyWbv+ZPz+6bm34sP3FdY9auLNYP+wbLtb/97pD6tYe7/1+cd1dw28V6yffs7JYP+bPHinWO6GpsNveJukNScOS9kVEbxVNAaheFXv234+IXRW8DoAW4jM7kESzYQ9JP7H9mO3lYz3B9nLbfbb7hrSnyc0BaFSzh/FLI2KH7TmS1tv+ZUQ8PPoJEbFa0mpJOsI90eT2ADSoqT17ROyo3e6UdJ+kJVU0BaB6DYfd9gzbMw/cl3S2pM1VNQagWs0cxs+VdJ/tA69zR0T8qJKuJpkpxy0q1mP6tGJ9xxkfKtbfPqX+mHDPB8vjxT/9dHm8uZP+49czi/V/+OdzivWNJ95Rt/bi0NvFdVcNfLZY/9hPD75PpA2HPSK2Svp0hb0AaCGG3oAkCDuQBGEHkiDsQBKEHUiCS1wrMHzmZ4r16269sVj/5LT6l2JOZkMxXKz/9Q1fLdanvlUe/jr1nhV1azNf3ldcd/qu8tDcYX0bi/VuxJ4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnL0C05/ZUaw/9pv5xfonpw1U2U6lVvafUqxvfbP8U9S3LvxB3drr+8vj5HP/6T+L9VY6+C5gHR97diAJwg4kQdiBJAg7kARhB5Ig7EAShB1IwhHtG1E8wj1xss9q2/a6xeAlpxbru88p/9zzlCcPL9af+MYN77unA67d9bvF+qNnlMfRh197vViPU+v/APG2bxVX1YJlT5SfgPfYGBu0OwbHnMuaPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4exeYMvvDxfrwq4PF+ot31B8rf/r0NcV1l/z9N4v1OTd27ppyvH9NjbPbXmN7p+3No5b12F5v+7na7awqGwZQvYkcxt8q6d2z3l8paUNELJK0ofYYQBcbN+wR8bCkdx9Hnidpbe3+WknnV9sWgKo1+gXd3Ijol6Ta7Zx6T7S93Haf7b4h7WlwcwCa1fJv4yNidUT0RkTvNE1v9eYA1NFo2Adsz5Ok2u3O6loC0AqNhn2dpItr9y+WdH817QBolXF/N972nZLOlDTb9nZJV0taJelu25dKeknSha1scrIb3vVqU+sP7W58fvdPffkXxforN00pv8D+8hzr6B7jhj0iltUpcXYMcBDhdFkgCcIOJEHYgSQIO5AEYQeSYMrmSeC4K56tW7vkxPKgyb8dtaFYP+PCy4r1md9/pFhH92DPDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM4+CZSmTX7168cV131p3dvF+pXX3las/8UXLyjW478/WLc2/+9+XlxXbfyZ8wzYswNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEkzZnNzgH55arN9+9XeK9QVTD21425+6bUWxvujm/mJ939ZtDW97smpqymYAkwNhB5Ig7EAShB1IgrADSRB2IAnCDiTBODuKYuniYv2IVduL9Ts/8eOGt33sg39UrP/O39S/jl+Shp/b2vC2D1ZNjbPbXmN7p+3No5ZdY/tl25tqf+dW2TCA6k3kMP5WSeeMsfx7EbG49vdAtW0BqNq4YY+IhyUNtqEXAC3UzBd0K2w/WTvMn1XvSbaX2+6z3TekPU1sDkAzGg37TZIWSlosqV/Sd+s9MSJWR0RvRPRO0/QGNwegWQ2FPSIGImI4IvZLulnSkmrbAlC1hsJue96ohxdI2lzvuQC6w7jj7LbvlHSmpNmSBiRdXXu8WFJI2ibpaxFRvvhYjLNPRlPmzinWd1x0TN3axiuuL677gXH2RV9+8exi/fXTXi3WJ6PSOPu4k0RExLIxFt/SdFcA2orTZYEkCDuQBGEHkiDsQBKEHUiCS1zRMXdvL0/ZfJgPKdZ/HXuL9c9/8/L6r33fxuK6Byt+ShoAYQeyIOxAEoQdSIKwA0kQdiAJwg4kMe5Vb8ht/2mLi/UXLixP2XzC4m11a+ONo4/nhsGTivXD7u9r6vUnG/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE4+yTnHtPKNaf/VZ5rPvmpWuL9dMPLV9T3ow9MVSsPzK4oPwC+8f9dfNU2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMsx8Epi44qlh/4ZKP1a1dc9FdxXW/cPiuhnqqwlUDvcX6Q9efUqzPWlv+3Xm807h7dtvzbT9oe4vtp21/u7a8x/Z628/Vbme1vl0AjZrIYfw+SSsj4jhJp0i6zPbxkq6UtCEiFknaUHsMoEuNG/aI6I+Ix2v335C0RdKRks6TdOBcyrWSzm9RjwAq8L6+oLN9tKSTJG2UNDci+qWRfxAkzamzznLbfbb7hrSnyXYBNGrCYbd9uKQfSro8InZPdL2IWB0RvRHRO03TG+kRQAUmFHbb0zQS9Nsj4t7a4gHb82r1eZJ2tqZFAFUYd+jNtiXdImlLRFw3qrRO0sWSVtVu729Jh5PA1KN/u1h//ffmFesX/e2PivU/+dC9xXorrewvD4/9/F/qD6/13PpfxXVn7WdorUoTGWdfKukrkp6yvam27CqNhPxu25dKeknShS3pEEAlxg17RPxM0piTu0s6q9p2ALQKp8sCSRB2IAnCDiRB2IEkCDuQBJe4TtDUeR+tWxtcM6O47tcXPFSsL5s50FBPVVjx8mnF+uM3LS7WZ/9gc7He8wZj5d2CPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJJFmnH3vH5R/tnjvnw4W61cd80Dd2tm/9VZDPVVlYPjturXT160srnvsX/2yWO95rTxOvr9YRTdhzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSaQZZ992fvnftWdPvKdl277xtYXF+vUPnV2se7jej/uOOPbaF+vWFg1sLK47XKxiMmHPDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJOCLKT7DnS7pN0kc1cvny6oi43vY1kv5Y0iu1p14VEfUv+pZ0hHviZDPxK9AqG2ODdsfgmCdmTOSkmn2SVkbE47ZnSnrM9vpa7XsR8Z2qGgXQOhOZn71fUn/t/hu2t0g6stWNAajW+/rMbvtoSSdJOnAO5grbT9peY3tWnXWW2+6z3TekPc11C6BhEw677cMl/VDS5RGxW9JNkhZKWqyRPf93x1ovIlZHRG9E9E7T9OY7BtCQCYXd9jSNBP32iLhXkiJiICKGI2K/pJslLWldmwCaNW7YbVvSLZK2RMR1o5bPG/W0CySVp/ME0FET+TZ+qaSvSHrK9qbasqskLbO9WFJI2ibpay3oD0BFJvJt/M8kjTVuVxxTB9BdOIMOSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQxLg/JV3pxuxXJP3PqEWzJe1qWwPvT7f21q19SfTWqCp7OyoiPjJWoa1hf8/G7b6I6O1YAwXd2lu39iXRW6Pa1RuH8UAShB1IotNhX93h7Zd0a2/d2pdEb41qS28d/cwOoH06vWcH0CaEHUiiI2G3fY7tZ2w/b/vKTvRQj+1ttp+yvcl2X4d7WWN7p+3No5b12F5v+7na7Zhz7HWot2tsv1x77zbZPrdDvc23/aDtLbaftv3t2vKOvneFvtryvrX9M7vtKZKelfRZSdslPSppWUT8oq2N1GF7m6TeiOj4CRi2T5f0pqTbIuKE2rJ/lDQYEatq/1DOiogruqS3ayS92elpvGuzFc0bPc24pPMlfVUdfO8KfX1RbXjfOrFnXyLp+YjYGhF7Jd0l6bwO9NH1IuJhSYPvWnyepLW1+2s18j9L29XprStERH9EPF67/4akA9OMd/S9K/TVFp0I+5GSfjXq8XZ113zvIeknth+zvbzTzYxhbkT0SyP/80ia0+F+3m3cabzb6V3TjHfNe9fI9OfN6kTYx5pKqpvG/5ZGxGckfU7SZbXDVUzMhKbxbpcxphnvCo1Of96sToR9u6T5ox5/XNKODvQxpojYUbvdKek+dd9U1AMHZtCt3e7scD//r5um8R5rmnF1wXvXyenPOxH2RyUtsr3A9iGSviRpXQf6eA/bM2pfnMj2DElnq/umol4n6eLa/Ysl3d/BXt6hW6bxrjfNuDr83nV8+vOIaPufpHM18o38C5L+shM91OnrE5KeqP093eneJN2pkcO6IY0cEV0q6cOSNkh6rnbb00W9/bukpyQ9qZFgzetQb6dp5KPhk5I21f7O7fR7V+irLe8bp8sCSXAGHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4k8X+zhHFo7nUhhwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(xb[0].view(28, 28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "c40ec589",
   "metadata": {},
   "outputs": [],
   "source": [
    "model, opt = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "a9090723",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.13, accuracy:0.96\n",
      "loss:0.11, accuracy:0.96\n",
      "loss:0.07, accuracy:0.98\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epoches):\n",
    "    for xb, yb in train_dl:\n",
    "        pred = model(xb)\n",
    "        loss = F.cross_entropy(pred, yb)\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        opt.zero_grad()\n",
    "    \n",
    "    report(loss, pred, yb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f33f4dc7",
   "metadata": {},
   "source": [
    "#### random sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "1e214f8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from itertools import islice\n",
    "import fastcore.all as fc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "6047e8a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sampler():\n",
    "    def __init__(self, ds, shuffle=False):\n",
    "        self.n = len(ds)\n",
    "        self.shuffle = shuffle\n",
    "    \n",
    "    def __iter__(self):\n",
    "        res = list(range(self.n))\n",
    "        if self.shuffle:\n",
    "            random.shuffle(res)\n",
    "        return iter(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "d90f09da",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = Sampler(train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "c65b920b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(iter(s))[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "b5202c75",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = Sampler(train_ds, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "30aab427",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[29118, 3418, 43685, 23876]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(iter(s))[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "7af0fa82",
   "metadata": {},
   "outputs": [],
   "source": [
    "fc.chunked??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "3b38b048",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = torch.arange(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "639e17a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "d0411215",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[tensor(0), tensor(1), tensor(2)], [tensor(3), tensor(4), tensor(5)]]"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(fc.chunked(t, 3, drop_last=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "b1029d25",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BatchSampler():\n",
    "    def __init__(self, sampler, bs, drop_last=False):\n",
    "        fc.store_attr()\n",
    "    \n",
    "    def __iter__(self):\n",
    "        yield from fc.chunked(self.sampler, self.bs, drop_last = self.drop_last)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "9883426d",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = BatchSampler(s, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "d5b20614",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.stack??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "ba10ff62",
   "metadata": {},
   "outputs": [],
   "source": [
    "def collate(b):\n",
    "    xs, ys = zip(*b)\n",
    "    return torch.stack(xs), torch.stack(ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "023295d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataLoader():\n",
    "    def __init__(self, ds, batches, collate_fn=collate):\n",
    "        fc.store_attr()\n",
    "    \n",
    "    def __iter__(self):\n",
    "        yield from (self.collate_fn(self.ds[i] for i in b) for b in self.batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "0474fec8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_samp = BatchSampler(Sampler(train_ds, shuffle=True), bs)\n",
    "val_samp = BatchSampler(Sampler(val_ds, shuffle=True), bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "6b421edc",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(train_ds, batches=train_samp)\n",
    "val_dl = DataLoader(val_ds, batches=val_samp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "6d7e4e98",
   "metadata": {},
   "outputs": [],
   "source": [
    "xb, yb = next(iter(train_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "412dc483",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([50, 784]), torch.Size([50]))"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xb.shape, yb.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "dd882ec2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1abc92fd0>"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOPElEQVR4nO3df6zV9X3H8dcLBFTACf4kgKVaU2u2Fd2tujA7F2pDjQ2arFtpQ1jS7Labbu2maY2tq9uaxfWHnV2cDVZTunU6a2WQaVvdXRdrVqkXS/lRqFCDBSGg4ipaReC+98f9slzxns+5nN/wfj6Sm3PO932+5/vOCS++55zP9/v9OCIE4Ng3rtsNAOgMwg4kQdiBJAg7kARhB5I4rpMbm+hJcbwmd3KTQCqv6RW9Hvs8Wq2psNteIOk2SeMlfS0ibik9/3hN1sWe38wmARSsioGatYY/xtseL+l2Se+TdL6kRbbPb/T1ALRXM9/ZL5K0JSKejojXJd0raWFr2gLQas2EfaakbSMeb6+WvYHtftuDtgf3a18TmwPQjGbCPtqPAG869jYilkZEX0T0TdCkJjYHoBnNhH27pNkjHs+StKO5dgC0SzNhf0LSubbfanuipA9KWtmatgC0WsNDbxFxwPa1kr6n4aG3uyNiQ8s6A9BSTY2zR8RDkh5qUS8A2ojDZYEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiiqVlcMzlu1syatd2Xn1Vc97TlPy3WD/7vLxvq6WgwdOkFNWvb//xAcd35c54q1jfc9JvF+sTvPlGsZ9NU2G1vlbRX0kFJByKirxVNAWi9VuzZfy8inm/B6wBoI76zA0k0G/aQ9LDt1bb7R3uC7X7bg7YH92tfk5sD0KhmP8bPi4gdtk+X9IjtTRHx6MgnRMRSSUsl6SRPjya3B6BBTe3ZI2JHdbtb0nJJF7WiKQCt13DYbU+2PfXQfUnvlbS+VY0BaK1mPsafIWm57UOv868R8d2WdNWDNn7uzJq1zZffXlz3xb95tVi/asPiYv2FvZOL9ZJ3zXqmWP/irO8U6/X2BkN16if68Zq1EzyxuO6CTQvLr73u2WK9PIqfT8Nhj4inJb2zhb0AaCOG3oAkCDuQBGEHkiDsQBKEHUiCU1zHaPpjtYeJXpxfHlqbNu6EYv36cx4u1le8UPs00WZ96tkFxfo4lQ96HJKL9ce/8xs1a2eu2l9cd9J//rhYP3CAwbUjwZ4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JwROcuHnOSp8fFnt+x7XXMwKxi+aHzVhbrl173p8X61HtrnyYKjLQqBvRS7Bn14Af27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOezt4A/+WvF+rr7y+dtT/9Y+XLPBx+cWqwP7d1brAMSe3YgDcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9haI1RuK9f/YW57sdsW5Dxbrl/z+NcX6qfevr1ljDB6H1N2z277b9m7b60csm277Edubq9tp7W0TQLPG8jH+65IOnzbkBkkDEXGupIHqMYAeVjfsEfGopD2HLV4oaVl1f5mkq1rbFoBWa/QHujMiYqckVben13qi7X7bg7YH92tfg5sD0Ky2/xofEUsjoi8i+iZoUrs3B6CGRsO+y/YMSapud7euJQDt0GjYV0paUt1fImlFa9oB0C51rxtv+x5Jl0k6VdIuSZ+V9O+S7pN0lqRfSPpARBz+I96bHLPXja/j6c//drG+6cO3N/X6X/3lW2rWNr96RnHdNS+Ur3l/8Wlbi/Vvrb2wWH/7F35VsxabthTXDeZfP2Kl68bXPagmIhbVKOVLLXAU43BZIAnCDiRB2IEkCDuQBGEHkmDK5g4Yf3L5UtObPndesf5X71lerM+dtO2IexqrcS7/+3jHhAnl9TXqKJAk6f1PXVlcd9/fzijWj/uv1cV6RkzZDICwA1kQdiAJwg4kQdiBJAg7kARhB5JgnB1NqXf67vg5L9esPXjxHcV1f/Dq2cX6nTddXaxP+daqYv1YxDg7AMIOZEHYgSQIO5AEYQeSIOxAEoQdSIJxdnTNqwsvKtaXfeXWYn187VPlJUkf+svratYm339sjsEzzg6AsANZEHYgCcIOJEHYgSQIO5AEYQeSqDuLK9AuJ6z4UbF+5fmfLNY3/Nk/FeuX3/SDmrUf/XBOcd0Dz+4o1o9Gdffstu+2vdv2+hHLbrb9rO011d8V7W0TQLPG8jH+65IWjLL8yxExt/p7qLVtAWi1umGPiEcl7elALwDaqJkf6K61vbb6mD+t1pNs99setD24X/ua2ByAZjQa9jsknSNprqSdkr5U64kRsTQi+iKib4ImNbg5AM1qKOwRsSsiDkbEkKQ7JZVPXwLQdQ2F3fbIuXSvlrS+1nMB9Ia64+y275F0maRTbW+X9FlJl9meKykkbZX00fa1iKxm/X35nPPzLlhcrG+Yt6xm7fw/+d3iunM+c+yNs9cNe0QsGmXxXW3oBUAbcbgskARhB5Ig7EAShB1IgrADSXCKK3rX0MFi+cAzU8rrz6td2n9WvkO32bMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs+OoNWVbnTmb8Qbs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZcdR6eXZ0u4WjCnt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYOGHfiieUnDA2Vy6+91sJujh0HJ5evK19y/M+Ob2EnR4e6e3bbs21/3/ZG2xtsf7xaPt32I7Y3V7fT2t8ugEaN5WP8AUnXRcQ7JF0i6Rrb50u6QdJARJwraaB6DKBH1Q17ROyMiCer+3slbZQ0U9JCScuqpy2TdFWbegTQAkf0A53tOZIukLRK0hkRsVMa/g9B0uk11um3PWh7cL/yza8F9Ioxh932FEnflvSJiHhprOtFxNKI6IuIvgma1EiPAFpgTGG3PUHDQf9mRDxQLd5le0ZVnyFpd3taBNAKdYfebFvSXZI2RsStI0orJS2RdEt1u6ItHR4DnvvwO4v1ve95pVg/e8lTxfrROjQ3/rTTivUtf/G2Yv1/rvxCsd6/bUHN2pyvbiqu2/igXu8ayzj7PEmLJa2zvaZadqOGQ36f7Y9I+oWkD7SlQwAtUTfsEfGYpFpX45/f2nYAtAuHywJJEHYgCcIOJEHYgSQIO5AEp7h2wLRN5XHwr914Z7G+fNWFxfq/DFxas/a2+35VXFePry2Wxx1fPhV03LSTi/UX3z2nZq3/rx+oWZOkxVO/V6w/se+EYn3rp99es3bcC6uL6x6L2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKO6Ny0tyd5elxsTpQ73N4/vKRY/4db/rFY/62J42vW9sWB4rof+vn7i/ULTt5WrH/m1PXFesnLUb5M2Y/3TS7Wr/+7jxbrp9z1wyPu6Wi3Kgb0UuwZ9SxV9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATns/eAqf/2eLF+/b5rivVXTq89zh5X7mmop0M+NvO/i/V3rV5UrL/4/NSatfNuK18vf+gnG4v1U5RvHL0Z7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIm657Pbni3pG5LOlDQkaWlE3Gb7Zkl/LOm56qk3RsRDpdfifHagvUrns4/loJoDkq6LiCdtT5W02vYjVe3LEfHFVjUKoH3GMj/7Tkk7q/t7bW+UNLPdjQForSP6zm57jqQLJK2qFl1re63tu21Pq7FOv+1B24P7Vb4MEYD2GXPYbU+R9G1Jn4iIlyTdIekcSXM1vOf/0mjrRcTSiOiLiL4JmtR8xwAaMqaw256g4aB/MyIekKSI2BURByNiSNKdki5qX5sAmlU37LYt6S5JGyPi1hHLZ4x42tWSGr/MKIC2G8uv8fMkLZa0zvaaatmNkhbZnispJG2VVL6uL4CuGsuv8Y9JGm3crjimDqC3cAQdkARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgibqXkm7pxuznJD0zYtGpkp7vWANHpld769W+JHprVCt7e0tEnDZaoaNhf9PG7cGI6OtaAwW92luv9iXRW6M61Rsf44EkCDuQRLfDvrTL2y/p1d56tS+J3hrVkd66+p0dQOd0e88OoEMIO5BEV8Jue4Htn9neYvuGbvRQi+2tttfZXmN7sMu93G17t+31I5ZNt/2I7c3V7ahz7HWpt5ttP1u9d2tsX9Gl3mbb/r7tjbY32P54tbyr712hr468bx3/zm57vKSnJF0uabukJyQtioifdrSRGmxvldQXEV0/AMP2uyW9LOkbEfHr1bLPS9oTEbdU/1FOi4hP9UhvN0t6udvTeFezFc0YOc24pKsk/ZG6+N4V+voDdeB968ae/SJJWyLi6Yh4XdK9khZ2oY+eFxGPStpz2OKFkpZV95dp+B9Lx9XorSdExM6IeLK6v1fSoWnGu/reFfrqiG6EfaakbSMeb1dvzfcekh62vdp2f7ebGcUZEbFTGv7HI+n0LvdzuLrTeHfSYdOM98x718j0583qRthHm0qql8b/5kXEhZLeJ+ma6uMqxmZM03h3yijTjPeERqc/b1Y3wr5d0uwRj2dJ2tGFPkYVETuq292Slqv3pqLedWgG3ep2d5f7+X+9NI33aNOMqwfeu25Of96NsD8h6Vzbb7U9UdIHJa3sQh9vYnty9cOJbE+W9F713lTUKyUtqe4vkbSii728Qa9M411rmnF1+b3r+vTnEdHxP0lXaPgX+Z9L+nQ3eqjR19mSflL9beh2b5Lu0fDHuv0a/kT0EUmnSBqQtLm6nd5Dvf2zpHWS1mo4WDO61NvvaPir4VpJa6q/K7r93hX66sj7xuGyQBIcQQckQdiBJAg7kARhB5Ig7EAShB1IgrADSfwfPeRp1uDTDAAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(xb[0].view(28, 28))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f59575f6",
   "metadata": {},
   "source": [
    "#### multiprocessing dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "7e61b268",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.multiprocessing as mp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "7dbebc2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.]]),\n",
       " tensor([5, 0, 4]))"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ds[[0, 1, 2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "008a36a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 0.]]),\n",
       " tensor([5, 4, 1]))"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ds.__getitem__([0, 9, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "35fca190",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]]), tensor([5, 1]))\n",
      "(tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
      "        [0., 0., 0.,  ..., 0., 0., 0.]]), tensor([9, 2]))\n"
     ]
    }
   ],
   "source": [
    "for o in map(train_ds.__getitem__, ([0, 3], [4, 5])):\n",
    "             print(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa19f681",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python_main",
   "language": "python",
   "name": "python_main"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}