encoderlstmdecoder.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/btknzn/TiagoRobotSkillLearning/blob/main/encoderlstmdecoder.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5EhaxNcsPbqX",
        "outputId": "4b3ec745-d3d2-4b68-f559-826523d20f00"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.25.1-py3-none-any.whl (5.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.8 MB 31.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Collecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.11.1-py3-none-any.whl (182 kB)\n",
            "\u001b[K     |████████████████████████████████| 182 kB 77.4 MB/s \n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 70.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.8.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.11.1 tokenizers-0.13.2 transformers-4.25.1\n"
          ]
        }
      ],
      "source": [
        "pip install transformers\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "reEz9H6ARAey"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import torch\n",
        "import torchvision\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import torchvision.datasets as dset\n",
        "from torch import nn\n",
        "#df = pd.read_csv('PlacingBack/data.csv')\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pds\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data import DataLoader\n",
        "import torchvision.datasets as dset\n",
        "from torchvision.io import read_image\n",
        "import cv2\n",
        "from sklearn.preprocessing import Normalizer\n",
        "import torchvision.transforms.functional as TF\n",
        "def read_image_dir(image_folder):\n",
        "    image_folder  = image_folder\n",
        "    lst = os.listdir(image_folder) # your directory path\n",
        "    number_files = len(lst)\n",
        "    images = []\n",
        "    for i in range(1,number_files):\n",
        "        folder = image_folder+\"/label\"+str(i)+\".jpg\"\n",
        "        img = cv2.imread(folder)\n",
        "        img.resize([img.shape[2],img.shape[0],img.shape[1]])\n",
        "        images.append(img)\n",
        "    return images\n",
        "\n",
        "def read_image_dir_b(image_folder):\n",
        "    image_folder  = image_folder\n",
        "    lst = os.listdir(image_folder) # your directory path\n",
        "    number_files = len(lst)\n",
        "    images = []\n",
        "    for i in range(1,number_files):\n",
        "        folder = image_folder+\"/label\"+str(i)+\".jpg\"\n",
        "        img = cv2.imread(folder,0)\n",
        "        img.resize(1,img.shape[0],img.shape[1])\n",
        "        img = img\n",
        "        images.append(img)\n",
        "    return images\n",
        "\n",
        "class MyDataset(Dataset):\n",
        " \n",
        "  def __init__(self,file_name,image_folder_normal,image_folder_depth):\n",
        "    df=pd.read_csv(file_name)\n",
        "    self.image_folder_norma = image_folder_normal\n",
        "    image = read_image_dir(image_folder_normal)\n",
        "    image = torch.asarray(image).float()\n",
        "    #correct it for complex situation\n",
        "    R_means = 188.3424\n",
        "    R_std = 54.67417\n",
        "    G_means = 177.37274\n",
        "    G_std = 66.43382\n",
        "    B_means = 169.08133\n",
        "    B_std = 72.04574\n",
        "    normalized_image = TF.normalize(image,[R_means, G_means, B_means], [R_std, G_std, B_std])\n",
        "    image = torch.asarray(normalized_image).float()\n",
        "    imageDs = read_image_dir_b(image_folder_depth)\n",
        "    D_std =  1.165845\n",
        "    D_means = 1.2093502\n",
        "    imageDs = torch.asarray(imageDs).float()\n",
        "    normalized_disparity = TF.normalize(imageDs,[D_means], [D_std])\n",
        "    imageDs = torch.asarray(normalized_disparity).float()\n",
        "\n",
        "    X1_MIN = 0.04995837\n",
        "    X1_MAX = 0.7562845\n",
        "    arm_joint1 = (df.arm_joint1-X1_MIN)/(X1_MAX-X1_MIN)\n",
        "    X2_MIN = -0.43724102\n",
        "    X2_MAX = 1.0187681\n",
        "    arm_joint2 = (df.arm_joint2-X2_MIN)/(X2_MAX-X2_MIN)\n",
        "    X3_MIN = -3.0051005\n",
        "    X3_MAX = -0.7699633\n",
        "    arm_joint3 = (df.arm_joint3-X3_MIN)/(X3_MAX-X3_MIN)\n",
        "\n",
        "    X4_MIN = 0.5167487\n",
        "    X4_MAX = 2.0607007\n",
        "    arm_joint4 = (df.arm_joint4-X4_MIN)/(X4_MAX-X4_MIN)\n",
        "    X5_MIN = -2.0750659\n",
        "    X5_MAX = 2.0743227\n",
        "    arm_joint5 = (df.arm_joint5-X5_MIN)/(X5_MAX-X5_MIN)\n",
        "\n",
        "    X6_MIN = -1.3956555\n",
        "    X6_MAX = 1.3958309\n",
        "    arm_joint6 = (df.arm_joint6-X6_MIN)/(X6_MAX-X6_MIN)\n",
        "    \n",
        "    X8_MIN = 0.022458013\n",
        "    X8_MAX = 0.045057185\n",
        "    gripper_joint1 = (df.gripper_joint1-X8_MIN)/(X8_MAX-X8_MIN)\n",
        "\n",
        "    X9_MIN = 0.022458013\n",
        "    X9_MAX = 0.045057185\n",
        "    gripper_joint2 = (df.gripper_joint2-X9_MIN)/(X9_MAX-X9_MIN)\n",
        "\n",
        "    X7_MIN = -1.8210393\n",
        "    X7_MAX = 2.0740623\n",
        "    arm_joint7= (df.arm_joint7-X7_MIN)/(X7_MAX-X7_MIN)\n",
        "\n",
        "\n",
        "    X10_MIN = -2.1165963e-06\n",
        "    X10_MAX = 0.3500109\n",
        "    torso_joint = (df.torso_joint-X10_MIN)/(X10_MAX-X10_MIN)\n",
        "\n",
        "    X11_MIN = -0.00033559513\n",
        "    X11_MAX = 0.0006243193\n",
        "    torsohead_joint1 = (df.torso_head1-X11_MIN)/(X11_MAX-X11_MIN)\n",
        "\n",
        "    X12_MIN = -0.7685144\n",
        "    X12_MAX = 0.0014590342  \n",
        "    torsohead_joint2 = (df.torso_head2-X12_MIN)/(X12_MAX-X12_MIN)\n",
        "\n",
        "   \n",
        "\n",
        "   \n",
        "   \n",
        "\n",
        "\n",
        "    \n",
        "\n",
        "    #1 Feature Number\n",
        "    self.arm_joint1 = arm_joint1.values\n",
        "    #self.arm_joint1 = df.arm_joint1.values\n",
        "    #2 Feature Number\n",
        "    self.arm_joint2 = arm_joint2.values\n",
        "    #self.arm_joint2 = df.arm_joint2.values\n",
        "    #3 Feature Number\n",
        "    self.arm_joint3 = arm_joint3.values\n",
        "    #self.arm_joint3 = df.arm_joint3.values\n",
        "    #4 Feature Number\n",
        "    self.arm_joint4 = arm_joint4.values\n",
        "    #self.arm_joint4 = df.arm_joint4.values\n",
        "    #5 Feature Number\n",
        "    self.arm_joint5 = arm_joint5.values\n",
        "    #self.arm_joint5 = df.arm_joint5.values\n",
        "    #6 Feature Number\n",
        "    self.arm_joint6 = arm_joint6.values\n",
        "    #self.arm_joint6 = df.arm_joint6.values\n",
        "    #7 Feature Number\n",
        "    self.arm_joint7 = arm_joint7.values\n",
        "    #self.arm_joint7 = df.arm_joint7.values\n",
        "    #8 Feature Number\n",
        "    self.gripper_joint1 = gripper_joint1.values\n",
        "    #self.gripper_joint1 = df.gripper_joint1.values\n",
        "    #9 Feature Number\n",
        "    self.gripper_joint2 = gripper_joint2.values\n",
        "    #self.gripper_joint2 = df.gripper_joint2.values\n",
        "    #10 Feature Number\n",
        "    self.torso_joint = torso_joint.values\n",
        "    #self.torso_joint = df.torso_joint.values\n",
        "    #11 Feature Number\n",
        "    self.torsohead_joint1 = torsohead_joint1.values\n",
        "    #self.torsohead_joint1 = df.torso_head1.values\n",
        "    #12 Feature Number\n",
        "    self.torsohead_joint2 = torsohead_joint2.values\n",
        "    #self.torsohead_joint2 = df.torso_head2.values\n",
        "\n",
        "    self.image = image\n",
        "    #self.image = read_image_dir(image_folder_normal)\n",
        "    #self.image =  torch.asarray(self.image).float()\n",
        "    self.imageDs = imageDs\n",
        "    #self.imageDs = read_image_dir_b(image_folder_depth)\n",
        "    #self.imageDs = torch.asarray(self.imageDs).float()\n",
        "\n",
        "\n",
        "  def __len__(self):\n",
        "    return \n",
        "   \n",
        "  def __getitem__(self,idx):\n",
        "    currentRobot = [self.arm_joint1[idx],self.arm_joint2[idx],self.arm_joint3[idx],self.arm_joint4[idx],self.arm_joint5[idx],self.arm_joint6[idx],self.arm_joint7[idx],self.gripper_joint1[idx],self.gripper_joint2[idx],self.torso_joint[idx],self.torsohead_joint1[idx],self.torsohead_joint2[idx]]\n",
        "    if idx >= len(self.arm_joint1)-2:   \n",
        "      nextActionRobot = [self.arm_joint1[idx],self.arm_joint2[idx],self.arm_joint3[idx],self.arm_joint4[idx],self.arm_joint5[idx],self.arm_joint6[idx],self.arm_joint7[idx],self.gripper_joint1[idx],self.gripper_joint2[idx],self.torso_joint[idx],self.torsohead_joint1[idx],self.torsohead_joint2[idx]]\n",
        "      x = torch.Tensor(currentRobot)\n",
        "      x = x.reshape((1,x.shape[0]))\n",
        "      image_current = self.image[idx]\n",
        "      image_next = self.image[idx]\n",
        "      depth_current = self.imageDs[idx]\n",
        "      depth_next = self.imageDs[idx]\n",
        "      y = torch.Tensor(nextActionRobot)\n",
        "      y = y.reshape((1,y.shape[0]))\n",
        "      return x,y,image_current,image_next,depth_current,depth_next\n",
        "\n",
        "    else:\n",
        "      image_current = self.image[idx] \n",
        "      depth_current = self.imageDs[idx]\n",
        "      idx = idx+1\n",
        "      nextActionRobot = [self.arm_joint1[idx],self.arm_joint2[idx],self.arm_joint3[idx],self.arm_joint4[idx],self.arm_joint5[idx],self.arm_joint6[idx],self.arm_joint7[idx],self.gripper_joint1[idx],self.gripper_joint2[idx],self.torso_joint[idx],self.torsohead_joint1[idx],self.torsohead_joint2[idx]]\n",
        "      x = torch.Tensor(currentRobot)\n",
        "      x = x.reshape((1,x.shape[0]))\n",
        "      y = torch.Tensor(nextActionRobot)\n",
        "      y = y.reshape((1,y.shape[0]))\n",
        "      image_next = self.image[idx]\n",
        "      depth_next = self.imageDs[idx]\n",
        "      return x,y,image_current,image_next,depth_current,depth_next\n",
        "\n",
        "\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KSltzjxsNn12"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import torch\n",
        "import torchvision\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import torchvision.datasets as dset\n",
        "from torch import nn\n",
        "#df = pd.read_csv('PlacingBack/data.csv')\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pds\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data import DataLoader,IterableDataset\n",
        "import torchvision.datasets as dset\n",
        "from torchvision.io import read_image\n",
        "import cv2\n",
        "from sklearn.preprocessing import Normalizer\n",
        "\n",
        "\n",
        "\n",
        "class DatasetItar(IterableDataset):\n",
        "  def __init__(self,file_name,image_folder_normal,image_folder_depth):\n",
        "    self.data = MyDataset(file_name,image_folder_normal,image_folder_depth)\n",
        "  def __iter__(self):\n",
        "        return iter(self.data)\n",
        "\n",
        "\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fshtaNXnRPlJ",
        "outputId": "9c665adf-4b2f-46d0-ea00-616519494515"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TmxeyPEKRfhz",
        "outputId": "2d354cdb-6606-4daf-d8f3-e6aa6cc22eca"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/drive/MyDrive/datasets\n"
          ]
        }
      ],
      "source": [
        "cd /content/drive/MyDrive/datasets/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Fv5FooIKR2v4"
      },
      "outputs": [],
      "source": [
        "\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "import  torch\n",
        "import torchvision\n",
        "from torch import nn\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import transforms\n",
        "import torchvision.datasets as dset\n",
        "from torchvision.utils import save_image\n",
        "import torchvision.utils as vutils\n",
        "from torchsummary import summary\n",
        "from IPython import display\n",
        "import torchvision.models as models\n",
        "from torchvision import datasets, models, transforms\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torchvision\n",
        "from torch import nn\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import transforms\n",
        "import torchvision.datasets as dset\n",
        "from torchvision.utils import save_image\n",
        "import torchvision.utils as vutils\n",
        "from torchsummary import summary\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "\n",
        "\n",
        "nb_channls=1\n",
        "image_size1 = 96\n",
        "image_size2 = 128 \n",
        "def to_img(x):\n",
        "    x = 0.5 * (x + 1)\n",
        "    x = x.clamp(0, 1)\n",
        "    x = x.view(x.size(0), nb_channls, image_size1, image_size2)\n",
        "    return x\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0uZH3d5gSMjv"
      },
      "outputs": [],
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "\"\"\"resnet18basedAutoencoder.ipynb\n",
        "\n",
        "Automatically generated by Colaboratory.\n",
        "\n",
        "Original file is located at\n",
        "    https://colab.research.google.com/drive/1TkVvsE39DDP-FzpxqKN2UzbWDu-5Wd57\n",
        "\"\"\"\n",
        "\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torchvision\n",
        "from torch import nn\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import transforms\n",
        "import torchvision.datasets as dset\n",
        "from torchvision.utils import save_image\n",
        "import torchvision.utils as vutils\n",
        "from torchsummary import summary\n",
        "from IPython import display\n",
        "import torchvision.models as models\n",
        "from torchvision import datasets, models, transforms\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torchvision\n",
        "from torch import nn\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import transforms\n",
        "import torchvision.datasets as dset\n",
        "from torchvision.utils import save_image\n",
        "import torchvision.utils as vutils\n",
        "from torchsummary import summary\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "\n",
        "\n",
        "nb_channls=3\n",
        "image_size1 = 96\n",
        "image_size2 = 128 \n",
        "def to_img(x):\n",
        "    x = 0.5 * (x + 1)\n",
        "    x = x.clamp(0, 1)\n",
        "    x = x.view(x.size(0), nb_channls, image_size1, image_size2)\n",
        "    return x\n",
        "\n",
        "\n",
        "class autoencoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(autoencoder, self).__init__()\n",
        "        self.encoder = nn.Sequential(\n",
        "            nn.Conv2d(3, 8, kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(8),\n",
        "            nn.Conv2d(8,16,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(16),\n",
        "            nn.Conv2d(16,32,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(32),\n",
        "            nn.Conv2d(32,16,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            ) \n",
        "        \n",
        "        self.decoder = nn.Sequential(\n",
        "            nn.BatchNorm2d(16),\n",
        "            nn.ConvTranspose2d(16,32,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(32),\n",
        "            nn.ConvTranspose2d(32,16,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(16),\n",
        "            nn.ConvTranspose2d(16,8,kernel_size=5,stride=2,output_padding=1),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(8),\n",
        "            nn.ConvTranspose2d(8,3,kernel_size=5,stride=2,output_padding=1),\n",
        "            nn.Sigmoid(),\n",
        "            nn.BatchNorm2d(3)\n",
        "            )    \n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.encoder(x)\n",
        "        x = self.decoder(x)\n",
        "        return x\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vRpTsdIDc70R"
      },
      "outputs": [],
      "source": [
        "class autoencoderBLACK(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(autoencoderBLACK, self).__init__()\n",
        "        self.encoder = nn.Sequential(\n",
        "            nn.Conv2d(1, 8, kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(8),\n",
        "            nn.Conv2d(8,16,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(16),\n",
        "            nn.Conv2d(16,32,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(32),\n",
        "            nn.Conv2d(32,16,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            ) \n",
        "        \n",
        "        self.decoder = nn.Sequential(\n",
        "            nn.BatchNorm2d(16),\n",
        "            nn.ConvTranspose2d(16,32,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(32),\n",
        "            nn.ConvTranspose2d(32,16,kernel_size=5,stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(16),\n",
        "            nn.ConvTranspose2d(16,8,kernel_size=5,stride=2,output_padding=1),\n",
        "            nn.ReLU(),\n",
        "            nn.BatchNorm2d(8),\n",
        "            nn.ConvTranspose2d(8,1,kernel_size=5,stride=2,output_padding=1),\n",
        "            nn.Sigmoid(),\n",
        "            nn.BatchNorm2d(1)\n",
        "            )    \n",
        "    \n",
        "    def forward(self,x):\n",
        "        x = self.encoder(x)\n",
        "        x = self.decoder(x)\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JsY6io35ciEw"
      },
      "outputs": [],
      "source": [
        "autoencoder_COLOR = autoencoder()\n",
        "autoencoder_COLOR.load_state_dict(torch.load(\"/content/drive/MyDrive/datasets/AECOLOR.pth\"))\n",
        "\n",
        "autoencoderBLACK = autoencoderBLACK()\n",
        "\n",
        "autoencoderBLACK.load_state_dict(torch.load(\"/content/drive/MyDrive/datasets/AEBLACK.pth\"))\n",
        "\n",
        "autoencoder_COLOR.requires_grad_=False\n",
        "autoencoder.requires_grad_=False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IbuoncnISZYI"
      },
      "outputs": [],
      "source": [
        "\n",
        "device = torch.device(\"cuda:0\" if (torch.cuda.is_available()) else \"cpu\")\n",
        "class LSTMPredictor(nn.Module):\n",
        "    def __init__(self,n_hidden=4096):\n",
        "        super(LSTMPredictor,self).__init__()\n",
        "        self.n_hidden = n_hidden\n",
        "        self.lstm1 = nn.LSTMCell(492,self.n_hidden)\n",
        "        self.lstm2 = nn.LSTMCell(self.n_hidden,self.n_hidden)\n",
        "        self.lstm3 = nn.LSTMCell(self.n_hidden,self.n_hidden)\n",
        "        self.lstm4 = nn.LSTMCell(self.n_hidden,self.n_hidden)\n",
        "        self.linear = nn.Linear(self.n_hidden,12)\n",
        "        self.encoder = autoencoder_COLOR.encoder\n",
        "        self.encoderB = autoencoderBLACK.encoder\n",
        "\n",
        "    def forward(self,x,image_current,depth_current,future = 0):\n",
        "        outputs = []\n",
        "        n_samples = 1\n",
        "        latent_RGB = self.encoder(image_current)\n",
        "        latent_D  = self.encoderB(depth_current)\n",
        "        flattenx = torch.flatten(x,start_dim=1)\n",
        "        latentImage = torch.flatten(latent_RGB,start_dim=1)\n",
        "        latentDepth = torch.flatten(latent_D,start_dim=1)\n",
        "        lstm_input = torch.cat((latentImage,latentDepth,flattenx),1)\n",
        "        lstm_input = lstm_input[:,None,:]\n",
        "        h_t = torch.zeros(n_samples,self.n_hidden,dtype=torch.float32).to(device)\n",
        "        c_t = torch.zeros(n_samples,self.n_hidden,dtype=torch.float32).to(device)\n",
        "        h_t2 = torch.zeros(n_samples,self.n_hidden,dtype=torch.float32).to(device)\n",
        "        c_t2 = torch.zeros(n_samples,self.n_hidden,dtype=torch.float32).to(device)\n",
        "        h_t3 = torch.zeros(n_samples,self.n_hidden,dtype=torch.float32).to(device)\n",
        "        c_t3 = torch.zeros(n_samples,self.n_hidden,dtype=torch.float32).to(device)\n",
        "        h_t4 = torch.zeros(n_samples,self.n_hidden,dtype=torch.float32).to(device)\n",
        "        c_t4 = torch.zeros(n_samples,self.n_hidden,dtype=torch.float32).to(device)\n",
        "        for input_t in lstm_input.split(1,dim=0):\n",
        "            h_t , c_t = self.lstm1(input_t[0],(h_t,c_t))\n",
        "            h_t2 , c_t2 = self.lstm2(h_t,(h_t,c_t))\n",
        "            h_t3 , c_t3 = self.lstm2(h_t,(h_t2,c_t2))\n",
        "            h_t4 , c_t4 = self.lstm2(h_t,(h_t3,c_t3))\n",
        "            output = self.linear(h_t4)\n",
        "            outputs.append(output)\n",
        "        for i in range(future):\n",
        "            h_t , c_t = self.lstm1(output,(h_t,c_t))\n",
        "            h_t2 , c_t2 = self.lstm2(h_t,(h_t2,c_t2))\n",
        "            output = self.linear(h_t2)\n",
        "            outputs.append(output)\n",
        "\n",
        "        outputs = torch.cat(outputs,dim=0)\n",
        "        return outputs\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1L5jYK1ONcNV",
        "outputId": "5bbe6b35-61e7-4ba3-8a13-f5c79612a0f6"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-4-f82e967b5693>:53: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:230.)\n",
            "  image = torch.asarray(image).float()\n"
          ]
        }
      ],
      "source": [
        "datasets_partial = []\n",
        "data_folder = '/content/drive/MyDrive/datasets'\n",
        "lst = os.listdir(data_folder)\n",
        "number_files = len(lst)\n",
        "completeMotionFolder = []\n",
        "for i in range(0,number_files):\n",
        "  folderCompleteMotion = str(i+1)\n",
        "  if  folderCompleteMotion in lst:\n",
        "    completeMotionFolder.append(data_folder+\"/\"+folderCompleteMotion)\n",
        "          \n",
        "      \n",
        "datasets = []            \n",
        "for i in range(0,11):\n",
        "  folder = completeMotionFolder[i]\n",
        "  lst = os.listdir(folder)\n",
        "  zeroCSV = folder +\"/0.csv\"\n",
        "  depthZERO = folder+'/depth/0'\n",
        "  imageZERO = folder + '/Image/0'\n",
        "  dataset = [zeroCSV,imageZERO,depthZERO] \n",
        "  datasets.append(dataset)\n",
        "\n",
        "train_loaders = []\n",
        "for i in range(0,len(datasets)):\n",
        "  CSV = datasets[i][0]\n",
        "  depth = datasets[i][1]\n",
        "  image = datasets[i][2]\n",
        "  dataset = DatasetItar(CSV,depth,image)\n",
        "  train_loader = DataLoader(dataset,shuffle=False,batch_size=32)\n",
        "  train_loaders.append(train_loader) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZbZaLUupPll9",
        "outputId": "4782b34c-0884-4668-c76f-b1ea8a9be7d7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['/content/drive/MyDrive/datasets/11/0.csv', '/content/drive/MyDrive/datasets/11/Image/0', '/content/drive/MyDrive/datasets/11/depth/0']\n"
          ]
        }
      ],
      "source": [
        "print(datasets[10])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bdBd_Uc2zovo"
      },
      "outputs": [],
      "source": [
        "model = LSTMPredictor().to(device)\n",
        "model.load_state_dict(torch.load(\"/content/drive/MyDrive/datasets/lasttry.pth\"))\n",
        "criterion = nn.MSELoss()\n",
        "learning_rate = 1e-4\n",
        "weight_decay = 1e-7"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "8QKmG2xBOXuh",
        "outputId": "8c9d003c-5ff2-422d-997b-44a30f61168d"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([32, 1, 12])) that is different to the input size (torch.Size([32, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([4, 1, 12])) that is different to the input size (torch.Size([4, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([17, 1, 12])) that is different to the input size (torch.Size([17, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([7, 1, 12])) that is different to the input size (torch.Size([7, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([9, 1, 12])) that is different to the input size (torch.Size([9, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([8, 1, 12])) that is different to the input size (torch.Size([8, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([27, 1, 12])) that is different to the input size (torch.Size([27, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([29, 1, 12])) that is different to the input size (torch.Size([29, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([13, 1, 12])) that is different to the input size (torch.Size([13, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.8/dist-packages/torch/nn/modules/loss.py:536: UserWarning: Using a target size (torch.Size([3, 1, 12])) that is different to the input size (torch.Size([3, 12])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.epochnumberloss: tensor(0.0161, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "1.epochnumberloss: tensor(0.0112, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "2.epochnumberloss: tensor(0.0102, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "3.epochnumberloss: tensor(0.0096, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "4.epochnumberloss: tensor(0.0090, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "5.epochnumberloss: tensor(0.0085, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "6.epochnumberloss: tensor(0.0081, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "7.epochnumberloss: tensor(0.0077, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "8.epochnumberloss: tensor(0.0074, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "9.epochnumberloss: tensor(0.0073, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "10.epochnumberloss: tensor(0.0072, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "11.epochnumberloss: tensor(0.0068, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "12.epochnumberloss: tensor(0.0066, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "13.epochnumberloss: tensor(0.0065, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "14.epochnumberloss: tensor(0.0063, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "15.epochnumberloss: tensor(0.0063, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "16.epochnumberloss: tensor(0.0063, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "17.epochnumberloss: tensor(0.0061, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "18.epochnumberloss: tensor(0.0061, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "19.epochnumberloss: tensor(0.0060, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "20.epochnumberloss: tensor(0.0059, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "21.epochnumberloss: tensor(0.0059, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "22.epochnumberloss: tensor(0.0059, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "23.epochnumberloss: tensor(0.0059, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "24.epochnumberloss: tensor(0.0059, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "25.epochnumberloss: tensor(0.0060, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "26.epochnumberloss: tensor(0.0061, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "27.epochnumberloss: tensor(0.0060, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "28.epochnumberloss: tensor(0.0058, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "29.epochnumberloss: tensor(0.0057, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "30.epochnumberloss: tensor(0.0057, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "31.epochnumberloss: tensor(0.0056, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "32.epochnumberloss: tensor(0.0055, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "33.epochnumberloss: tensor(0.0055, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "34.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "35.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "36.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "37.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "38.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "39.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "40.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "41.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "42.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "43.epochnumberloss: tensor(0.0055, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "44.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "45.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "46.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "47.epochnumberloss: tensor(0.0055, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "48.epochnumberloss: tensor(0.0055, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "49.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "50.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "51.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "52.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "53.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "54.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "55.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "56.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "57.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "58.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "59.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "60.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "61.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "62.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "63.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "64.epochnumberloss: tensor(0.0055, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "65.epochnumberloss: tensor(0.0055, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "66.epochnumberloss: tensor(0.0055, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "67.epochnumberloss: tensor(0.0056, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "68.epochnumberloss: tensor(0.0056, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "69.epochnumberloss: tensor(0.0054, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "70.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "71.epochnumberloss: tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "72.epochnumberloss: tensor(0.0052, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "73.epochnumberloss: tensor(0.0051, device='cuda:0', grad_fn=<DivBackward0>)\n",
            "74.epochnumberloss: tensor(0.0051, device='cuda:0', grad_fn=<DivBackward0>)\n"
          ]
        }
      ],
      "source": [
        "n_epoch = 1000\n",
        "model.encoder.requires_grad_=False\n",
        "model.encoderB.requires_grad_=False\n",
        "learning_rate = 1e-4\n",
        "weight_decay = 1e-7\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n",
        "for epoch in range(0,n_epoch):\n",
        "  currentLoss = []\n",
        "  for j in range(0,len(train_loaders)):\n",
        "    train_loader = train_loaders[j]\n",
        "    for data in (train_loader):\n",
        "      x = data[0][:,:,:].to(device)\n",
        "      x_next = data[1][:,:,:].to(device)\n",
        "      image_current = data[2].to(device)\n",
        "      image_next = data[3].to(device)\n",
        "      depth_current = data[4].to(device)\n",
        "      depth_next = data[5].to(device)\n",
        "      train_input = x\n",
        "      train_target = x_next\n",
        "      optimizer.zero_grad()\n",
        "      out = model(train_input,image_current,depth_current)\n",
        "      loss = criterion(out,train_target)+(1/1000)*criterion(image_current,image_next)+(1/1000)*criterion(depth_current,depth_next)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      currentLoss.append(loss)\n",
        "\n",
        "\n",
        "  print(str(epoch)+\".epochnumber\"+ \"loss:\", sum(currentLoss)/len(currentLoss))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LKYdzYrDfdSl"
      },
      "outputs": [],
      "source": [
        "n_epoch = 1000\n",
        "model.encoder.requires_grad_= True\n",
        "model.encoderB.requires_grad_= True\n",
        "learning_rate = 1e-6\n",
        "weight_decay = 1e-8\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n",
        "for epoch in range(0,n_epoch):\n",
        "  currentLoss = []\n",
        "  for j in range(0,len(train_loaders)):\n",
        "    train_loader = train_loaders[j]\n",
        "    for data in (train_loader):\n",
        "      x = data[0][:,:,:].to(device)\n",
        "      x_next = data[1][:,:,:].to(device)\n",
        "      image_current = data[2].to(device)\n",
        "      image_next = data[3].to(device)\n",
        "      depth_current = data[4].to(device)\n",
        "      depth_next = data[5].to(device)\n",
        "      train_input = x\n",
        "      train_target = x_next\n",
        "      optimizer.zero_grad()\n",
        "      out = model(train_input,image_current,depth_current)\n",
        "      loss = criterion(out,train_target)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      currentLoss.append(loss)\n",
        "  print(\"loss:\", sum(currentLoss)/len(currentLoss))\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "W5lBaFJzfhvY"
      },
      "outputs": [],
      "source": [
        "n_epoch = 1000\n",
        "model.encoder.requires_grad_= False\n",
        "model.encoderB.requires_grad_= False\n",
        "learning_rate = 1e-8\n",
        "weight_decay = 1e-12\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n",
        "for epoch in range(0,n_epoch):\n",
        "  currentLoss = []\n",
        "  for j in range(0,len(train_loaders)*0+1):\n",
        "    train_loader = train_loaders[10]\n",
        "    for data in (train_loader):\n",
        "      x = data[0][:,:,:].to(device)\n",
        "      x_next = data[1][:,:,:].to(device)\n",
        "      image_current = data[2].to(device)\n",
        "      image_next = data[3].to(device)\n",
        "      depth_current = data[4].to(device)\n",
        "      depth_next = data[5].to(device)\n",
        "      train_input = x\n",
        "      train_target = x_next\n",
        "      optimizer.zero_grad()\n",
        "      out = model(train_input,image_current,depth_current)\n",
        "      loss = criterion(out,train_target)\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      currentLoss.append(loss)\n",
        "  print(str(epoch)+\".epochnumber\"+ \"loss:\", sum(currentLoss)/len(currentLoss))\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fxSN6E1DSMT0"
      },
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0OqgX8laO4UH"
      },
      "outputs": [],
      "source": [
        "x1 = []\n",
        "x1_pred = []\n",
        "x2 = []\n",
        "x2_pred = []\n",
        "x3 = []\n",
        "x3_pred = []\n",
        "x4 = []\n",
        "x4_pred = []\n",
        "x5 = []\n",
        "x5_pred = []\n",
        "x6 = []\n",
        "x6_pred = []\n",
        "x7 = []\n",
        "x7_pred = []\n",
        "x8 = []\n",
        "x8_pred = []\n",
        "x9= []\n",
        "x9_pred = []\n",
        "x10 = []\n",
        "x10_pred = []\n",
        "x11 = []\n",
        "x11_pred = []\n",
        "x12 = []\n",
        "x12_pred = []\n",
        "\n",
        "\n",
        "train_loaders = []\n",
        "for i in range(0,len(datasets)):\n",
        "  CSV = datasets[i][0]\n",
        "  depth = datasets[i][1]\n",
        "  image = datasets[i][2]\n",
        "  dataset = DatasetItar(CSV,depth,image)\n",
        "  train_loader = DataLoader(dataset,shuffle=False,batch_size=1)\n",
        "  train_loaders.append(train_loader) \n",
        "\n",
        "for j in range(0,len(train_loaders)):\n",
        "  train_loader = train_loaders[10]\n",
        "  for data in (train_loader):\n",
        "    x = data[0][:,:,:].to(device)\n",
        "    x_next = data[1][:,:,:].to(device)\n",
        "    image_current = data[2].to(device)\n",
        "    image_next = data[3].to(device)\n",
        "    depth_current = data[4].to(device)\n",
        "    depth_next = data[5].to(device)\n",
        "    train_input = x\n",
        "    train_target = x_next\n",
        "    out = model(train_input,image_current,depth_current)\n",
        "    x1.append(torch.flatten(x[:,:,0]).tolist())\n",
        "    x2.append(torch.flatten(x[:,:,1]).tolist())\n",
        "    x3.append(torch.flatten(x[:,:,2]).tolist())\n",
        "    x4.append(torch.flatten(x[:,:,3]).tolist())\n",
        "    x5.append(torch.flatten(x[:,:,4]).tolist())\n",
        "    x6.append(torch.flatten(x[:,:,5]).tolist())\n",
        "    x7.append(torch.flatten(x[:,:,6]).tolist())\n",
        "    x8.append(torch.flatten(x[:,:,7]).tolist())\n",
        "    x9.append(torch.flatten(x[:,:,8]).tolist())\n",
        "    x10.append(torch.flatten(x[:,:,9]).tolist())\n",
        "    x11.append(torch.flatten(x[:,:,10]).tolist())\n",
        "    x12.append(torch.flatten(x[:,:,11]).tolist())\n",
        "    x1_pred.append(out[:,0].tolist())\n",
        "    x2_pred.append(out[:,1].tolist())\n",
        "    x3_pred.append(out[:,2].tolist())\n",
        "    x4_pred.append(out[:,3].tolist())\n",
        "    x5_pred.append(out[:,4].tolist())\n",
        "    x6_pred.append(out[:,5].tolist())\n",
        "    x7_pred.append(out[:,6].tolist())\n",
        "    x8_pred.append(out[:,7].tolist())\n",
        "    x9_pred.append(out[:,8].tolist())\n",
        "    x10_pred.append(out[:,9].tolist())\n",
        "    x11_pred.append(out[:,10].tolist())\n",
        "    x12_pred.append(out[:,11].tolist())\n",
        "                \n",
        "                \n",
        "\n",
        "\n",
        "    \n",
        "\n",
        "x1 = np.array(x1)\n",
        "x2 = np.array(x2)\n",
        "x3 = np.array(x3)\n",
        "x4 = np.array(x4)\n",
        "x5 = np.array(x5)\n",
        "x6 = np.array(x6)\n",
        "x7 = np.array(x7)\n",
        "x8 = np.array(x8)\n",
        "x9 = np.array(x9)\n",
        "x10 = np.array(x10)\n",
        "x11 = np.array(x11)\n",
        "x12 = np.array(x12)\n",
        "\n",
        "x1_pred = np.array(x1_pred)\n",
        "x2_pred = np.array(x2_pred)\n",
        "x3_pred = np.array(x3_pred)\n",
        "x4_pred = np.array(x4_pred)\n",
        "x5_pred = np.array(x5_pred)\n",
        "x6_pred = np.array(x6_pred)\n",
        "x7_pred = np.array(x7_pred)\n",
        "x8_pred = np.array(x8_pred)\n",
        "x9_pred = np.array(x9_pred)\n",
        "x10_pred = np.array(x10_pred)\n",
        "x11_pred = np.array(x11_pred)\n",
        "x12_pred = np.array(x12_pred)\n",
        "\n",
        "\n",
        "t1 = np.arange(0.0, x1_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t1, x1, 'bo')\n",
        "plt.plot( t1, x1_pred, 'or')\n",
        "plt.savefig('outputs/x1axis.png')\n",
        "\n",
        "t2 = np.arange(0.0, x2_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t2, x2, 'bo')   \n",
        "plt.plot( t2, x2_pred, 'or')\n",
        "plt.savefig('outputs/x2axis.png')\n",
        "\n",
        "t3 = np.arange(0.0, x3_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t3, x3, 'bo')\n",
        "plt.plot( t3, x3_pred, 'or')\n",
        "plt.savefig('outputs/x1axis.png')\n",
        "\n",
        "\n",
        "t4 = np.arange(0.0, x4_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t4, x4, 'bo')\n",
        "plt.plot( t4, x4_pred, 'or')\n",
        "plt.savefig('outputs/x4axis.png')\n",
        "\n",
        "t5 = np.arange(0.0, x5_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t5, x5, 'bo')\n",
        "plt.plot( t5, x5_pred, 'or')\n",
        "plt.savefig('outputs/x5axis.png')\n",
        "\n",
        "t6 = np.arange(0.0, x6_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t6, x6, 'bo')\n",
        "plt.plot( t6, x6_pred, 'or')\n",
        "plt.savefig('outputs/x6axis.png')\n",
        "\n",
        "t7 = np.arange(0.0, x7_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t7, x7, 'bo')\n",
        "plt.plot( t7, x7_pred, 'or')\n",
        "plt.savefig('outputs/x7axis.png')   \n",
        "\n",
        "t8 = np.arange(0.0, x8_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t8, x8, 'bo')\n",
        "plt.plot( t8, x8_pred, 'or')\n",
        "plt.savefig('outputs/x8axis.png') \n",
        "\n",
        "t9 = np.arange(0.0, x9_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t9, x9, 'bo')\n",
        "plt.plot( t9, x9_pred, 'or')\n",
        "plt.savefig('outputs/x9axis.png') \n",
        "\n",
        "\n",
        "t10 = np.arange(0.0, x10_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t10, x10, 'bo')\n",
        "plt.plot( t10, x10_pred, 'or')\n",
        "plt.savefig('outputs/x10axis.png') \n",
        "\n",
        "t11 = np.arange(0.0, x11_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t11, x11, 'bo')\n",
        "plt.plot( t11, x11_pred, 'or')\n",
        "plt.savefig('outputs/x11axis.png') \n",
        "\n",
        "t12 = np.arange(0.0, x12_pred.shape[0], 1) \n",
        "plt.figure()\n",
        "plt.plot(t12, x12, 'bo')\n",
        "plt.plot( t12, x12_pred, 'or')\n",
        "plt.savefig('outputs/x12axis.png')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PinRdwAPyS2z"
      },
      "outputs": [],
      "source": [
        "torch.save(model.state_dict(), '/content/drive/MyDrive/datasets/lasttry.pth')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "mount_file_id": "1CmoEUntFzoK2CJgt4kTwnSlHFdcV5gde",
      "authorship_tag": "ABX9TyNx6OeeRWfOWwDPIOEXSAvL",
      "include_colab_link": true
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}