Best Model - Training - SegFormer_b5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install torch torchvision --index-url https://download.pytorch.org/whl/cu121"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install -U scikit-learn scipy matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install shapely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install -U albumentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install scikit-image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\vravula\\Downloads\\Cracks\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "from transformers import AdamW\n",
    "import torch\n",
    "from torch import nn\n",
    "from sklearn.metrics import accuracy_score\n",
    "from tqdm.notebook import tqdm\n",
    "import os\n",
    "from PIL import Image \n",
    "from transformers import SegformerForSemanticSegmentation, SegformerImageProcessor\n",
    "import pandas as pd\n",
    "import cv2\n",
    "import numpy as np\n",
    "import matplotlib as plt\n",
    "# import albumentations as aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "WIDTH = 512\n",
    "HEIGHT = 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#one hot encode labels\n",
    "def label_to_nparray(seg):\n",
    "    channels = 19\n",
    "    seg_0 = seg.shape[0]\n",
    "    seg_1 = seg.shape[1]\n",
    "    nparray = np.zeros([seg_0, seg_1, channels], dtype=np.uint8)\n",
    "    for idx in range(0,channels):\n",
    "        mask = seg == idx\n",
    "        nparray[:,:,idx] = mask\n",
    "\n",
    "        # if idx == 19:\n",
    "        #     idx_ignore = 100\n",
    "        #     mask = seg == idx_ignore\n",
    "        #     nparray[idx,:,:] = mask\n",
    "\n",
    "    return nparray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ImageSegmentationDataset(Dataset):\n",
    "    \"\"\"Image segmentation dataset.\"\"\"\n",
    "\n",
    "    def __init__(self, root_dir, feature_extractor, transforms=None, train=True):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            root_dir (string): Root directory of the dataset containing the images + annotations.\n",
    "            feature_extractor (SegFormerFeatureExtractor): feature extractor to prepare images + segmentation maps.\n",
    "            train (bool): Whether to load \"training\" or \"validation\" images + annotations.\n",
    "        \"\"\"\n",
    "        self.root_dir = root_dir\n",
    "        self.feature_extractor = feature_extractor\n",
    "        self.train = train\n",
    "        self.transforms = transforms\n",
    "\n",
    "        sub_path = \"train\" if self.train else \"validation\"\n",
    "        self.img_dir = os.path.join(self.root_dir, \"images\", sub_path)\n",
    "        print(\"img dir: \", self.img_dir) #debug\n",
    "\n",
    "        self.ann_dir = os.path.join(self.root_dir, \"annotations\", sub_path)\n",
    "        print(\"annotations dir: \", self.ann_dir) #debug\n",
    "        \n",
    "        # read images\n",
    "        image_file_names = []\n",
    "        for root, dirs, files in os.walk(self.img_dir):\n",
    "            image_file_names.extend(files)\n",
    "        self.images = sorted(image_file_names)\n",
    "\n",
    "        print(\"Imported Images \",len(self.images)) #debug\n",
    "        \n",
    "        # read annotations\n",
    "        annotation_file_names = []\n",
    "        for root, dirs, files in os.walk(self.ann_dir):\n",
    "            annotation_file_names.extend(files)\n",
    "        self.annotations = sorted(annotation_file_names)\n",
    "        \n",
    "        print(\"Imported Annotations \",len(self.annotations)) #debug\n",
    "\n",
    "        assert len(self.images) == len(self.annotations), \"There must be as many images as there are segmentation maps\"\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.images)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        # print(self.img_dir) #debug\n",
    "        # print(self.images[idx]) #debug\n",
    "        \n",
    "        image = cv2.imread(os.path.join(self.img_dir, self.images[idx]))\n",
    "        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "        image = Image.fromarray(image)\n",
    "\n",
    "        # print(self.ann_dir) #debug\n",
    "        # print(self.annotations[idx]) #debug\n",
    "        \n",
    "        segmentation_map = cv2.imread(os.path.join(self.ann_dir, self.annotations[idx]))\n",
    "        segmentation_map = cv2.cvtColor(segmentation_map, cv2.COLOR_BGR2GRAY)\n",
    "        segmentation_map = Image.fromarray(segmentation_map)\n",
    "\n",
    "        # segmentation_map = Image.open()\n",
    "\n",
    "        if self.transforms is not None:\n",
    "            augmented_image = self.transforms(image)\n",
    "            augmented_mask = self.transforms(segmentation_map)\n",
    "\n",
    "            # randomly crop + pad both image and segmentation map to same size\n",
    "            encoded_inputs = self.feature_extractor(augmented_image, augmented_mask, return_tensors=\"pt\")\n",
    "        else:\n",
    "            encoded_inputs = self.feature_extractor(image, segmentation_map, return_tensors=\"pt\")\n",
    "\n",
    "        for k,v in encoded_inputs.items():\n",
    "            encoded_inputs[k].squeeze_() # remove batch dimension\n",
    "\n",
    "        return encoded_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision.transforms as transforms\n",
    "\n",
    "# Define data augmentation\n",
    "transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=1),\n",
    "    transforms.RandomVerticalFlip(p=1),\n",
    "    # transforms.ColorJitter(brightness=.5, hue=.3)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "img dir:  C:/Users/vravula/Downloads/Cracks/data_megazord\\images\\train\n",
      "annotations dir:  C:/Users/vravula/Downloads/Cracks/data_megazord\\annotations\\train\n",
      "Imported Images  20805\n",
      "Imported Annotations  20805\n",
      "img dir:  C:/Users/vravula/Downloads/Cracks/data_megazord\\images\\validation\n",
      "annotations dir:  C:/Users/vravula/Downloads/Cracks/data_megazord\\annotations\\validation\n",
      "Imported Images  2925\n",
      "Imported Annotations  2925\n"
     ]
    }
   ],
   "source": [
    "# C:\\Users\\vravula\\Downloads\\Cracks\\data\\dacl10k_v2_devphase\\resized\\data\\labels\\validation\n",
    "root_dir = 'C:/Users/vravula/Downloads/Cracks/data_megazord'\n",
    "\n",
    "# feature_extractor = SegformerImageProcessor(align=False, reduce_zero_label=False)\n",
    "feature_extractor = SegformerImageProcessor()\n",
    "\n",
    "train_dataset = ImageSegmentationDataset(root_dir=root_dir, feature_extractor=feature_extractor, transforms=None)\n",
    "valid_dataset = ImageSegmentationDataset(root_dir=root_dir, feature_extractor=feature_extractor, transforms=None, train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.ImageSegmentationDataset at 0x12785dc9590>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples: 20805\n",
      "Number of validation examples: 2925\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of training examples:\", len(train_dataset))\n",
    "print(\"Number of validation examples:\", len(valid_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_inputs = train_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 512, 512])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_inputs[\"pixel_values\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([512, 512])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_inputs[\"labels\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cv2.imwrite(\"test_label.png\",encoded_inputs[\"labels\"].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[182, 221],\n",
       "        [183, 215],\n",
       "        [183, 216],\n",
       "        ...,\n",
       "        [356, 287],\n",
       "        [356, 288],\n",
       "        [357, 287]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.nonzero(encoded_inputs[\"labels\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(True)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_inputs[\"labels\"].any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_inputs[\"labels\"].squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0, 10])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_inputs[\"labels\"].squeeze().unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = encoded_inputs[\"labels\"].numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1278843fc90>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAAGiCAYAAAC/NyLhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3CUlEQVR4nO3deXhTVf4G8DdLk24kpYUmFFooO2WVAiW4jlQLIuoIM6goVRGlFkYWEXFQRP1ZhRkXHBZXcIFBGEWUbcQiIFC2Clq2slhtgaYtYNONpk1yfn90iMYWSrokPc37eZ48D7n33NzvPQ15k3vPvVchhBAgIiKShNLbBRAREbmDwUVERFJhcBERkVQYXEREJBUGFxERSYXBRUREUmFwERGRVBhcREQkFQYXERFJhcFFRERS8VpwLVy4EB06dIC/vz/i4uKwd+9eb5VCREQS8Upwffrpp5g2bRrmzJmD77//Hn379kVCQgLy8/O9UQ4REUlE4Y2L7MbFxWHgwIH417/+BQBwOByIjIzE5MmT8fTTT3u6HCIikoja0yusqKhAeno6Zs2a5ZymVCoRHx+PtLS0GpexWq2wWq3O5w6HAxcuXEBYWBgUCkWj10xERA1LCIHi4mJERERAqXRv55/Hg+vcuXOw2+0wGAwu0w0GA44dO1bjMikpKZg7d64nyiMiIg/KyclBu3bt3FrG48FVF7NmzcK0adOczy0WC6KionAdboMafl6sjIiI6sKGSuzABrRo0cLtZT0eXK1atYJKpUJeXp7L9Ly8PBiNxhqX0Wq10Gq11aar4Qe1gsFFRCSd/42uqMvhHo+PKtRoNIiNjUVqaqpzmsPhQGpqKkwmk6fLISIiyXhlV+G0adOQmJiIAQMGYNCgQXjjjTdQWlqKhx56yBvlEBGRRLwSXGPGjEFBQQGee+45mM1m9OvXD5s2bao2YIOIiOiPvHIeV30VFRVBr9fjJtzJY1xERBKyiUpsxVpYLBbodDq3luW1ComISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCpuB9f27dsxcuRIREREQKFQ4IsvvnCZL4TAc889hzZt2iAgIADx8fE4ceKES5sLFy5g7Nix0Ol0CAkJwfjx41FSUlKvDSEiIt/gdnCVlpaib9++WLhwYY3z582bhwULFmDJkiXYs2cPgoKCkJCQgPLycmebsWPH4vDhw9i8eTPWrVuH7du349FHH637VhARkc9QCCFEnRdWKLBmzRrcddddAKp+bUVERGD69Ol48sknAQAWiwUGgwHLli3DPffcg6NHjyImJgb79u3DgAEDAACbNm3CbbfdhtOnTyMiIqLW9RYVFUGv1+Mm3Am1wq+u5RMRkZfYRCW2Yi0sFgt0Op1byzboMa6srCyYzWbEx8c7p+n1esTFxSEtLQ0AkJaWhpCQEGdoAUB8fDyUSiX27NlT4+tarVYUFRW5PIiIyDc1aHCZzWYAgMFgcJluMBic88xmM8LDw13mq9VqhIaGOtv8UUpKCvR6vfMRGRnZkGUTEZFEpBhVOGvWLFgsFucjJyfH2yUREZGXNGhwGY1GAEBeXp7L9Ly8POc8o9GI/Px8l/k2mw0XLlxwtvkjrVYLnU7n8iAiIt/UoMEVHR0No9GI1NRU57SioiLs2bMHJpMJAGAymVBYWIj09HRnmy1btsDhcCAuLq4hyyEiomZI7e4CJSUlOHnypPN5VlYWDh48iNDQUERFRWHKlCl46aWX0KVLF0RHR+PZZ59FRESEc+Rhjx49MGzYMEyYMAFLlixBZWUlJk2ahHvuueeqRhQSEZFvczu49u/fjz/96U/O59OmTQMAJCYmYtmyZXjqqadQWlqKRx99FIWFhbjuuuuwadMm+Pv7O5dZvnw5Jk2ahKFDh0KpVGLUqFFYsGBBA2wOERE1d/U6j8tbeB4XEZHcmsx5XERERI2NwUVERFJhcBERkVQYXEREJBUGFxERSYXBRUREUmFwERGRVBhcREQkFQYXERFJhcFFRERSYXAREZFUGFxERCQVBhcREUmFwUVERFJhcBERkVQYXEREJBUGFxERSYXBRUREUmFwERGRVBhcREQkFQYXERFJhcFFRERSYXAREZFUGFxERCQVBhcREUmFwUVERFJhcBERkVQYXEREJBUGFxERSYXBRUREUmFwERGRVBhcREQkFQYXERFJhcFFRERSYXAREZFUGFxERCQVBhcREUmFwUVERFJhcBERkVQYXEREJBUGFxERSYXBRUREUmFwERGRVBhcREQkFQYXERFJhcFFRERSYXAREZFUGFxERCQVBhcREUmFwUVERFJhcBERkVQYXERuUvbtAdvQWG+XQeSz1N4ugKipUYWFujzPfLYrEq496HzeM2gHwlQl2F7UDZu3XIMu8zIBAKK0DI7yck+WSuSTGFzk09QdolBwU1vnc4cKWP3cfBhUGuc0reIbqBTVd07c02I37PfvgnWsDQAQt+9BBKzVo0VOBdSp6Y1fvKSU/v6oNMVA9e333i6FJMXgIp9zfMkgqFpUAgC6tzVjb9fFf2gRfNWvpVIoEaioCrmMuBVAHLC8OAxz9t8BAGiZ6o/QD9IapG5ZXbxrEM7+pdL5XOtfgRk91+Gjv90Bv6/3N+q6bTfHwqFRQrNpX6OuhzxLIYQQ3i7CXUVFRdDr9bgJd0Kt8PN2OdSEqNu1hSNM53xeMCAEs5/+2KXNiEAL/BQqj9RzqrIEGRVGvHJiGFrOqnqvKi8Uw5Zz2iPr9wRVy5YQUUbnc3uQBolL1yFIaQUAdPQ7hz4a/2rL7bVWYvb9E6DYebBxClOqkD07Dv9+6HX8bAvD1NT70H1xMZTFF2H76efGWecVqDtEwZZ9BnDYPb7upsgmKrEVa2GxWKDT6Wpf4HcYXCS1szOGwK797fn1dxzA2+2a9i+cSWfisGVtLALzBMLebdq11qTwAROK2yucz1WxhVW/Nusg9aIKk5c+hg4rc2E/mdVQJQIA1G0j8MXer6p9SVlYGImFK0bCrxQwvr6rQdf5R9YRA5F/TdVn1N2jv8Oqjdeh7dZKaP7buL80ZcDgomZJoVZDof5tb3alKQaa58wubVZ1+RzByurf5mVwtKIM07NGo6A0GK3+ehpwOCCEgLBavVqXwk8DharqmJ5Cr4N1uT+0Kptz/vMd1mKQtmH/34375QYUxNvhKC1tsNdUd4jCFzvXXPbX9Tl7Kcad/AsqHSpoxlbAUWiBw2oF6vKRqFRBqanqk4L7r0H42F8AAA+23Ym/Bltcmn5U1AqLs25EyGgzYLc33oAepQoQjrptjwcwuKhZsN42EA7Nb9/kKx87j9U9P3Q+91co0EoV5I3SGt1pWwkAYG1JD3z6zHAo7ID/un2N/6GjUKB8xECI3x3tDn7iNN7ptAoAoALQRn31x/zq4/6fb8KFcS0b7JdXp33+WNR291W1zbWVwA5g+FtPoeUJG1r8kAdb1i9XXEYVFoqS6ztXLT9YhW/vmw8AaKFUQa8MqHWdl/7md704A633WeA4eOSqar0SdYcoFPer2m17ZlQl/I8GoO3WUih2/VDv125oDC5yUneIwoUhEdCtuLr/sN6i7NsDxya2+G2CAtg34vVmG0zuKnNUoOeGZMCugNqiQsenG2aXojD1xYlxv9u3qhI4fNtCBCo1l1/Ig0afisfFv6phyzXX3rgWg3+oxNzWh+u07O3Hh+Pw0UgYdiqh/+S3/0uqVmE4+mInAECQoRSHBi+vd50AMDu/N1Z8NwRdnzzo9i9uZa/uOJZc9cHftetZ/LfHOpf5y4vDMHvLKMTM/QU2c16D1NsQGFwEAFC1bo0em85jTOgebCvtXm2+1eGHnff1g7L4D7tjbHbYzpxt0FoUfhqoIgy/m6CAfnkxrtFnAwDaa85V24VCNStzVGBhYY+qv9/9/aC0lAIOUeMAD6W/P5SG1s7nR19ojaTYbc7nXbR5uCuoxCN119VD2dcjf0wIHHkFdd6NpgrR48bvzmJm2Il61ZJurcCW0h7O5y2U5ZgYcqZer3k5duHAW4Ud8c6/b0P0xzkQpWWwnzvvnK82GgBt1ReMgE8uIq5l1S/TSL8LuKfFr7W+/juWCFjsAVj9j1vRetNPXg8xjwVXSkoKPv/8cxw7dgwBAQEYMmQIXn31VXTr1s3Zpry8HNOnT8fKlSthtVqRkJCARYsWwWD47UMsOzsbSUlJ+PbbbxEcHIzExESkpKRArb660fkMrupUPbshcml2rQMT7MJRbVrqRS2eWjChxvYRmwtgP1r7f37bzbE43+u3b/LFHRw4PmaRa401nAtF7rn098u1l2HEa09B8Yc/p6WHDSfvWOJ8Lmuf24UDndc/hm7JP0BUVri9/Mk3BuPUX5fU3rCJsgsHJp6+HvuW93VOm5z0OR7UVX3BrO/f9R1LBBYtvgttV53yWoB5LLiGDRuGe+65BwMHDoTNZsMzzzyDQ4cO4ciRIwgKqtrFk5SUhPXr12PZsmXQ6/WYNGkSlEoldu7cCQCw2+3o168fjEYj5s+fj9zcXIwbNw4TJkzAyy+/fFV1MLhcqcJCEbnxYqOMppuQcy325kbV2i6p6/ZG+yZKvskuHOjyWRK6/G2P28ueWBCHn0a/3QhVNS+3Zd6GXzZ3QLuU/312eHAHnNd2FRYUFCA8PBzbtm3DDTfcAIvFgtatW2PFihUYPXo0AODYsWPo0aMH0tLSMHjwYGzcuBG33347zp496/wVtmTJEsycORMFBQXQaGrf187g+o06uj26/OcM3mjD4bXU/PTaPRZt73bvOFVlfCyWf/CmxwaVyO5XexlO2Pyw+tdBOPhEP/hl/AR7YePvxq9PcNXr96bFUrVxoaFV13ZLT09HZWUl4uPjnW26d++OqKgopKVVJXpaWhp69+7tsuswISEBRUVFOHy45jeo1WpFUVGRy4MAVddOCFvxK0OLmqz++8fUedkSRzmiJuS6vZzDT8nQckNLVSAGaf0w33gAmz9dipKbutW+kJfVObgcDgemTJmCa6+9Fr169QIAmM1maDQahISEuLQ1GAwwm83ONr8PrUvzL82rSUpKCvR6vfMRGRlZ17KbFUu/1vio/XZvl0FUo+j1E2Cc8CtufOxRXJ/8GPZaK2FxXLzq5ft/PBUOi3tfUhVqNXq98KO7pdL/XLPvHgTvbNgTwRtDnYMrOTkZhw4dwsqVKxuynhrNmjULFovF+cjJyWn0dTZ1FcMG4pP5//B2GUSXFfiTH+x5+fD/ai8C1+zBc92vw/WvTcctR0dibkFMrcuHHAOEzVZrOxcKJaaGp9axYt+WVVkCx7ehsBcUeLuUWtUpuCZNmoR169bh22+/Rbt27ZzTjUYjKioqUFhY6NI+Ly8PRqPR2SYvL6/a/EvzaqLVaqHT6Vwevqx85CBMWvApov24O4TkIaxWtHltF5RDc5D24DWIWfg4sm01D82fkHMtWu2/4PY6cp4cgFYqz1yHsrn5T3FfGN9o3EtgNRS3gksIgUmTJmHNmjXYsmULoqOjXebHxsbCz88Pqam/fePJzMxEdnY2TCYTAMBkMiEjIwP5+fnONps3b4ZOp0NMTO3fwnxdZXws3lzwFkYF8zgfNV33ZN2M9u8cu+x8ceAwIv9vFyYmPIQuHydhVYkeO8urxvaXOSqQuqcX7Icz3V5v6z+dvaqrVlB1Xyfd4O0SrppbtzVJTk7GihUrsHbtWrRo0cJ5TEqv1yMgIAB6vR7jx4/HtGnTEBoaCp1Oh8mTJ8NkMmHw4MEAgFtvvRUxMTF44IEHMG/ePJjNZsyePRvJycnQarVXWr3PKx85CK8t+Bf6sZ+oiTtfHgTl+dp36duPnkDHmSfw/sxoiCF34VSSEuP77qrTEHhh6ovhbeT4xdDU3JN1M/yO/AJZrlvvVnAtXlx136KbbrrJZfrSpUvx4IMPAgBef/11KJVKjBo1yuUE5EtUKhXWrVuHpKQkmEwmBAUFITExES+88EL9tqSZqxg2EH/750rEapvGpXmIrqTSroK7X68Uu35A513A5mE3QAP375919sagel8pwxfZhQPHVnaH4bw8oc9LPklAFaLHWz+sRyce0yIJZNtKMLH/nS6XK/KEM08PwaG/Laq9IbkYdOAvCL3rlzpdoaQ+vHYeF3mOQcWbVZM8HMWevR6iqnVrdBz+k0fX2RyctpUAq1t5PLTqi8Elgax3oxCg4C5CkkMbVQBOLu0BZZAHr/TfKgRfdtnkufU1E1vKOqDlh037ThI1YXBJYHDkz9JeLJV8j59ChZM3LcPPyzpCFaL3yDpPD29deyOq5uPHRjbZG01eCT8NiahRHL32YxxfFF17wwYw7dH/eGQ9zcngg6OhyWzY2xl5CoOLiBrN4RvfRdbKPlCFhTbaOlSdoxGqatr3GGtqcm0lKN0S7vV7ctUVg6uJqxg2EMNDee01kpNW4YfjN3yErCVtoWik8w+PzQ7FHUFljfLazdXz5lsQ8Q95hr//EYOriTtzk5p3Cibp7TO9C1XrVg3+ugq1Gkq/6jdHpcsrc1Qg8/le3i6jXhhcRNTogpX+MG04BWXfHg36upa/DMDRm95r0Nds7p7JG4KA7y5/OS4ZMLiIyCNmtzqGLu+fhKpzAw7YUFSNYqSrl/bmQDiKi71dRr0wuJowhVaLyhBZrh5GVLsFEfvwpy9+hDqyXe2Na6NQoCycH2HuuthaASgU3i6jXvhXb8LsA3sg6453vF0GUYOaEXoKvb48DcU1Pev1Oiq9Dtuf/GcDVeU7dk19DarQlt4uo14YXETkca8aDmLA0h+hblPzPfiuipqXQaszyfuOwUVEXvFSeAa6rDtX9xdYreW9t+ogWOkP/9XyXS3j9xhcROQ1//1qUJ2XbeXPk47r6tGI7bh4Z9373tsYXE2VQoFurx/1dhVEjcYuHOi4tPabTdbE/MQQvNR2QwNX5DuGBVqRa5J3NCaDqwm7P0zeM9uJGlN5uECUmvenq4+QXuehMoR7u4w6YXARkVd02/YwHOcuuL2cqmsn/GXEjkaoyLfs678KlV3beruMOmFwEZFXBKcFwlFa6vZytlbBeCk8oxEq8h0LCyPR/d3Hodp92Nul1IncYyKbMcvYOLRTfweAu0Oo+VlYGIlWGeV1WvbkGI4krI/BB0ej5eizaF+2C7KOLeQvriaqbLQF7bgPn5qpd45fB9XW791fUKHAxyMXNXg9vuCzEh2un/QYwiaUwVEm99X0+YuLiDzK4riIi0dC6rawhHfr9ba91kq8lD0StkeDEZi5BzZvF9QA+IuLiDzqaIUG0X/fXadl8yYPQUe13L8WPOlXexmSX/gbrDeaYc886e1yGgyDqwlS6XQICajb/n+i5sz2JwvacBd6rUoc5fiyNBC3PjcdoUvTvF1Og2NwNUG5D/TC9t5rvF0GUaN4KP3BOu3yU/aLQXz7zIYvqBnqs/YJLOzStVmGFsBjXE2T3HccILqidm/W7YoN5iEh2NhmfwNX07zcl/UnnHivO7p9+gOa832hGVxE5DFflgZCXVLh9oeqwk+D4k7N+aO4/o5WlKFgZnuE7khr1qEFcFchEXnQk58lwnHwiNvLqcJbIfMeDoO/kh6aQNy65DucfGMwVCF6b5fTqBhcRNTkOVrqvF2CFGaEnsKpvy5ByaehULcxQuGn8XZJjYLBRUQesddaiZBjdVs29pPDUCn4cXW1tvdegy/2r8epl2IBRfM7aM53QhNk3H4Bfzs70NtlEDWot/P+hJbL6jbKzU9hb+Bqmj8/hQo/jH0Tp/8Tg9JRcd4up0ExuJogx4/HsPV0Z2+XQdSg9psj67RcVooJ08PqcHkoQqBSg8Om5fjgtddQOioO6uj23i6pQTC4iKjRlTjK0fYhc52WtbVwIFjp38AV+ZaufkHY8dbbiF5lhrJPd2+XU28MLiJqdIUOGyDqMEhbqYJQ8/qEDeVfbffg+k8O4MSyWKkHbjC4iKjR3TV3BuyFFreXsw7rj2MjFzZCRb7rmVaZOHHLu/jpox4o+7Ocx74YXETU6FSVdVtOKBXQKvwathiCSqHE8Rs/xGcLXkPe34Z4uxy38coZRFehxFGOHeWNd1JnhNqCPprmeRzntK0E6ovu7yZU+GmQ8PI2bCrTNkJVTV+kuhA9NY1708xwVRDWTJ+HzUldsTa+H2xnzjbq+hoKg4t8zo8V5Ri9+1G3lrGfDUTnaXW7FcfVsA2NRc6E2n+WRLYqRGrMl41WR2O4cdtkdF69x/0FhQMffzYU2+buaviimrjzj5hQmlCCo9d+3OjrivYLxo7CzhDl8tyRgsFFUqgUdjj+cAU2uxAw/XMK/IrdO3ivtTgQXZcP0kakTk1HdOpVtDMaMOD2pKt6zV9vLMepoUvrWVn9CUfdToAVNhuifCy0FGo1cpMH4fNp89DJz3O3b/lhZS8Yz8vT1wyuJqr0Zz3KBlQgUCnvyJ+6euvX9si2hrpM2/VKHPTbfqrW1pif5lN3xbWZ8xD2Xt5Vta3QDwGGNnJBtfisRIduk040+4u+1pe6QxRKehnx4oJ30NtvO1qqeM+xK2FwNVGdp+7Bj3eoMLh5HvYAANx98hb8sL9TtendlhTAfvyUy7Rg7AavnSCfmV/dh07FjbeLtTn4Ze4QdL3pJ2zr8s7/pgR6dP3zL3RCeLpcd5VmcFGDsjgu4rSt+vQ7P5uKqI2ux3ACTp1D56zqH2oMqOYh11aCru+d59/zCn56xYT99/8TemXjDsK4kk9ODoJxx0Gvrb8uGFxUJ1mVJbjvSGK16UXfGRA5b2+16Z3se6rt0qsh36gZGfr2U4g8Ks9xE09TdY7GNdcd92poyYrBRbVaVaLHi++MdZmmLRQIe6/6BVN1OAXfOeJEl/PyuW6I+rqY74XLUAYFwf+DEqzqeBUjchpRVmUJdB/Jd8sYBpePsopKfFjUHg7x24ivC7ZgfHffNVBYK1zaKsorEJHDb850dezCgQ8yhqDTvgPeLqXJsvfpjH93eg+Ad0+uLnBoEbim+h6Spo7B5QNidt2Pi4WuozwUZSp0eWJvDSPy6njDJKL/OWMvQ+eHj/DX1mVUJAzApLdWNYkrgty361F0EvJ9wWBw+YCo/xMQB/Z7uwzyIHUbIx5+eINX1p3w3lOIqqjbfbeauuNLBqH79MNVT+x2ONw8aVdxTU88smANRgUXNUJ1V29uQQy+/r8b0HVzppSDZxhcRM2QCArAlJY/N/p6puQOwNmLrpfCivymtNmeWzfzhvUYfew4AOD9wn748oWbAQCBuVYor2Jk3plb9Bjb4nxjllir2fm98X1CGwTnyXuKCYOLiK5KurUCYz9+wmVap3eyYTt9xmWaAt79YG5srVRBAICZYScw880TAICPilphznd/BgC03qmu8U7P5ieG4NvJ8wEEeazWP5p0Jg6nxkfDkSf3IQEGVxOWXt4Bg/1zvF0GEY5XlmL641PQfqPrBzJPaagyTncO40a8CwD48ZZy7Hqq6sT6Nz+9E9HLcwEA4XfkOEPP045XlmL81GnQfZ8Lx89yhxbA4Gq6hMAXj8Uj+VPvX2uOfFeJoxyDdj+CsOVBCNzYtK7v2FT10fijj6bqV+jExxYBj3m3nvVl/kiZ+RiCPt/TbL5oMLiasmZ6nIDkYBcO9P9oKqL/vpvvRQBQKKBUyNMPduFArr0Mrz75KIK+aF5fOhhcROR00GrFqsKBAIDPv7wOHV/aD8HQAgCcWDAID+kWAVB5u5Ra5dtLcf2yJ9HpX6cQkCffeVq1YXAR+bjPSnR4asN9AIDQHxQIXVp1HKs9dvFcrN8RAXb4KZp+aAHA9R8+iQ7Ppkk7arA2DC4iH7S+zB//SL4fAKD51YrOe3kF96bgeGWp89+JR8bB/42WdXqd6C3pzfpLB4OrCfO7UIY3fu3gkfNxyLcU2HTQ/JcnpXvb3IIYbM7tDgCw2tRofZ8ZjrKqW4zoHD8DjlNXWPrymnNoAQyuJs1+5DgWfTUcU8Yt9nYpRNQAvigNxrPvjnM+b/ttMYL2ZQCoOrurue7aa2gMriau8/xMTL+lP/7Z5ntvl0Lk03r8PRu3zR9dr9dQWCsQ8fNvF6xu7r+MGguDq4mzn7+A9euG4JVH9klzYJioObLn5QN5+d4ugwAovV0A1a7j64dhFZW1NyT6H4Xjyt/llXBA4afxUDVEDYvBJQFRacP887HeLoMkIpSKK84f2yIfBZ93gLqN0UMVETUcBpcEHKWl2PSPG7xdBjUjKoUS6bGrULg0EGeeHuLtcojcwuCSRNi6Y+i/f4xby/xqL8PWi0ooKjlWydco37u6+0Tt7PM5Nj8+D9lzhnDXIUnDreBavHgx+vTpA51OB51OB5PJhI0bNzrnl5eXIzk5GWFhYQgODsaoUaOQl5fn8hrZ2dkYMWIEAgMDER4ejhkzZsBmay6Xfmw89l9/hWNzGGJ23Y8puQNqbX/HiWEwffAkUjr1geOQ/FeDJvcMDz901W3bqINx9LFF+Hl27e8roqbArVGF7dq1wyuvvIIuXbpACIEPP/wQd955Jw4cOICePXti6tSpWL9+PVavXg29Xo9Jkybh7rvvxs6dOwEAdrsdI0aMgNFoxK5du5Cbm4tx48bBz88PL7/8cqNsYHNieGsX8BaQ2bcH+puqricX/GczUnv9B0pUHdP4qkyHF/75ANpsPI32v+y60ssRudjx8HzcWD4D7V7dAzj4K52aLoWo5xU0Q0NDMX/+fIwePRqtW7fGihUrMHp01bkOx44dQ48ePZCWlobBgwdj48aNuP3223H27FkYDAYAwJIlSzBz5kwUFBRAo7m6XRVFRUXQ6/W4CXdCrfCrT/nSU+l0ULQIRt5t7eFQAcbPTsJeUODtssjL7jhyHskh7t/L7Vd7GX6yqfHkxMd5ZQ1qVDZRia1YC4vFAp1O59aydT7GZbfbsXLlSpSWlsJkMiE9PR2VlZWIj493tunevTuioqKQllZ10c60tDT07t3bGVoAkJCQgKKiIhw+fPiy67JarSgqKnJ5UBV7URFsZ84i7N00tF6SxtCiemmpCkSsVoM5iz7AxbsGebscohq5HVwZGRkIDg6GVqvFxIkTsWbNGsTExMBsNkOj0SAkJMSlvcFggNlsBgCYzWaX0Lo0/9K8y0lJSYFer3c+IiMj3S2biNxwU4ADOcMAhZrXKKCmx+3g6tatGw4ePIg9e/YgKSkJiYmJOHLkSGPU5jRr1ixYLBbnIyeHt7MnamzHRy6GNf4ab5dBVI3bX6c0Gg06d+4MAIiNjcW+ffvw5ptvYsyYMaioqEBhYaHLr668vDwYjVUnORqNRuzd63pTs0ujDi+1qYlWq4VWq3W3VCKqBz+FCuLK5zETeUW9z+NyOBywWq2IjY2Fn58fUlNTnfMyMzORnZ0Nk8kEADCZTMjIyEB+/m/X+9q8eTN0Oh1iYmLqWwoR/c+G22Nx29C/oNt347CprO5f+kraclchNT1ujSqcNWsWhg8fjqioKBQXF2PFihV49dVX8d///he33HILkpKSsGHDBixbtgw6nQ6TJ08GAOzaVTUs2263o1+/foiIiMC8efNgNpvxwAMP4JFHHnFrODxHFRJdPVW3zjg2Sw+Vxo5jN34AleLqv6+eqixBcuebISorGrFC8kX1GVXo1tep/Px8jBs3Drm5udDr9ejTp48ztADg9ddfh1KpxKhRo2C1WpGQkIBFixY5l1epVFi3bh2SkpJgMpkQFBSExMREvPDCC24VTURXz555El0eBBR+Glw3+nHoHjmNVd1Ww1+hhpZf/EhC9T6Pyxv4i4uo7pT+/lD4a5F7X0/0H/cjlkRuu+wtc3JtJRg3djKU3x3wcJXU3HnlPC4ikpOjvBz2QgvCF+3C2ZsdOGe/eNm2bdTBsD13wYPVEdWOwUXkw06+1xXhqsDLzs+2lcB/eoAHKyKqHYOLyFcN6o1H+3x3xcEa9x0ZB8ehEx4siqh2DC4iH5V9WwvMCD11xTbqN8N4wV1qchhcRD5I2ac7Nj8874ptotdPgP83P3ioIqKrx+Ai8kHHHwxBO3XwZefvLHcg4mslz9+iJonBReRjlEFBWHjH0iu2WX5+CIJX7/FQRUTuYXAR+Zjjb3fFsEDrZefbhQO//CXcgxURuYfBReRjlMorX3MgZseDcBSc91A1RO5jcBGRU7atBCHrguAoLfV2KUSXxeAi8iHKPt1xS5djl50/bO9EhHyc5sGKiNzHexYQ+ZCCgS2xse3KGucdrriItm/y2p/U9PEXFxEBAJ7NvoMX0yUpMLiICABQ+lQbb5dAdFUYXES+QqFAhV5R46zoDY9A+eNJDxdEVDcMLiIfoWrVCrunvVFt+s5yBwxb1XCUlXm+KKI6YHAR+RCVovovrsXmm6H/ZLcXqiGqGwYXkY9bErUJOX8f4u0yiK4ag4vIR4iyMtycMaba9GClPzrekgV1u7ZeqIrIfQwuIh/hKC1F5b8NKHNUv+L7uq4bUd7V6IWqiNzH4CLyIS0/2o1+Ox6pcd60d5Z7uBqiumFwEfkSISAcNf+37685h7zJPNZFTR+Di8jHdH2qAKkXVdWmh6q0sPSq9EJFRO5hcBH5mOIBbWFUlVSbnlauRdfH9nmhIiL3MLiIfIlCAfM95eipCag2a/KSiV4oiMh9DC4iH/LruMH4/rp3qk3/rESHyA28eSTJgcFF5COUQUFQ3ZOPYKW/y/SVxS3xr7+Ngf1wppcqI3IPg4vIRyiiIrC1T/V7cb2dfQM0m3hsi+TB4CLyEZ0//hlaheuNIn+1lyHgL4XeKYiojhhcRD7g1wdNSAzbWW16/3VT4Cgp9UJFRHXH4CJq5pQtWuDczVbEajUu07tuS0T3qT9C2GxeqoyobhhcRM1cxcCu+Cn+A5dphysuouXGQDjKy71UFVHdqb1dABE1IoUCfrPNOG377YTjUKUGG0p6IeSjNC8WRlR3DC6i5kwIKG47jwmId076eWZ/RC/PBZDlvbqI6oHBRdTM/XF3YNTcXbB7qRaihsBjXEREJBUGFxERSYXBRUREUmFwERGRVBhcREQkFQYXERFJhcFFRERSYXAREZFUGFxERCQVBhcREUmFwUVERFJhcBERkVQYXEREJBUGFxERSYXBRUREUmFwERGRVBhcREQkFQYXERFJhcFFRERSYXAREZFUGFxERCQVBhcREUmFwUVERFJhcBERkVQYXEREJBUGFxERSaVewfXKK69AoVBgypQpzmnl5eVITk5GWFgYgoODMWrUKOTl5bksl52djREjRiAwMBDh4eGYMWMGbDZbfUohIiIfUefg2rdvH95++2306dPHZfrUqVPx1VdfYfXq1di2bRvOnj2Lu+++2znfbrdjxIgRqKiowK5du/Dhhx9i2bJleO655+q+FURE5DPqFFwlJSUYO3Ys3n33XbRs2dI53WKx4P3338drr72Gm2++GbGxsVi6dCl27dqF3bt3AwC+/vprHDlyBJ988gn69euH4cOH48UXX8TChQtRUVHRMFtFRETNVp2CKzk5GSNGjEB8fLzL9PT0dFRWVrpM7969O6KiopCWlgYASEtLQ+/evWEwGJxtEhISUFRUhMOHD9e4PqvViqKiIpcHERH5JrW7C6xcuRLff/899u3bV22e2WyGRqNBSEiIy3SDwQCz2exs8/vQujT/0ryapKSkYO7cue6WSkREzZBbv7hycnLwxBNPYPny5fD392+smqqZNWsWLBaL85GTk+OxdRMRUdPiVnClp6cjPz8f/fv3h1qthlqtxrZt27BgwQKo1WoYDAZUVFSgsLDQZbm8vDwYjUYAgNForDbK8NLzS23+SKvVQqfTuTyIiMg3uRVcQ4cORUZGBg4ePOh8DBgwAGPHjnX+28/PD6mpqc5lMjMzkZ2dDZPJBAAwmUzIyMhAfn6+s83mzZuh0+kQExPTQJtFRETNlVvHuFq0aIFevXq5TAsKCkJYWJhz+vjx4zFt2jSEhoZCp9Nh8uTJMJlMGDx4MADg1ltvRUxMDB544AHMmzcPZrMZs2fPRnJyMrRabQNtFhERNVduD86ozeuvvw6lUolRo0bBarUiISEBixYtcs5XqVRYt24dkpKSYDKZEBQUhMTERLzwwgsNXQoRETVDCiGE8HYR7ioqKoJer8dNuBNqhZ+3yyEiIjfZRCW2Yi0sFovb4xZ4rUIiIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpIKg4uIiKTC4CIiIqkwuIiISCoMLiIikgqDi4iIpMLgIiIiqTC4iIhIKgwuIiKSCoOLiIikwuAiIiKpMLiIiEgqDC4iIpKKW8H1/PPPQ6FQuDy6d+/unF9eXo7k5GSEhYUhODgYo0aNQl5enstrZGdnY8SIEQgMDER4eDhmzJgBm83WMFtDRETNntrdBXr27IlvvvnmtxdQ//YSU6dOxfr167F69Wro9XpMmjQJd999N3bu3AkAsNvtGDFiBIxGI3bt2oXc3FyMGzcOfn5+ePnllxtgc4iIqLlzO7jUajWMRmO16RaLBe+//z5WrFiBm2++GQCwdOlS9OjRA7t378bgwYPx9ddf48iRI/jmm29gMBjQr18/vPjii5g5cyaef/55aDSa+m8RERE1a24f4zpx4gQiIiLQsWNHjB07FtnZ2QCA9PR0VFZWIj4+3tm2e/fuiIqKQlpaGgAgLS0NvXv3hsFgcLZJSEhAUVERDh8+fNl1Wq1WFBUVuTyIiMg3uRVccXFxWLZsGTZt2oTFixcjKysL119/PYqLi2E2m6HRaBASEuKyjMFggNlsBgCYzWaX0Lo0/9K8y0lJSYFer3c+IiMj3SmbiIiaEbd2FQ4fPtz57z59+iAuLg7t27fHqlWrEBAQ0ODFXTJr1ixMmzbN+byoqIjhRUTko+o1HD4kJARdu3bFyZMnYTQaUVFRgcLCQpc2eXl5zmNiRqOx2ijDS89rOm52iVarhU6nc3kQEZFvqldwlZSU4NSpU2jTpg1iY2Ph5+eH1NRU5/zMzExkZ2fDZDIBAEwmEzIyMpCfn+9ss3nzZuh0OsTExNSnFCIi8hFu7Sp88sknMXLkSLRv3x5nz57FnDlzoFKpcO+990Kv12P8+PGYNm0aQkNDodPpMHnyZJhMJgwePBgAcOuttyImJgYPPPAA5s2bB7PZjNmzZyM5ORlarbZRNpCIiJoXt4Lr9OnTuPfee3H+/Hm0bt0a1113HXbv3o3WrVsDAF5//XUolUqMGjUKVqsVCQkJWLRokXN5lUqFdevWISkpCSaTCUFBQUhMTMQLL7zQsFtFRETNlkIIIbxdhLuKioqg1+txE+6EWuHn7XKIiMhNNlGJrVgLi8Xi9rgFt09AbgouZa0NlYB0sUtERDZUAvjt89wdUgbX+fPnAQA7sMHLlRARUX0UFxdDr9e7tYyUwRUaGgqg6oK97m6wr7h0rltOTg5PH6gB++fK2D9Xxv65sqvpHyEEiouLERER4fbrSxlcSmXVKH69Xs83TS143tuVsX+ujP1zZeyfK6utf+r6w4P34yIiIqkwuIiISCpSBpdWq8WcOXN40vIVsI+ujP1zZeyfK2P/XFlj94+U53EREZHvkvIXFxER+S4GFxERSYXBRUREUmFwERGRVKQMroULF6JDhw7w9/dHXFwc9u7d6+2SPGL79u0YOXIkIiIioFAo8MUXX7jMF0LgueeeQ5s2bRAQEID4+HicOHHCpc2FCxcwduxY6HQ6hISEYPz48SgpKfHgVjSelJQUDBw4EC1atEB4eDjuuusuZGZmurQpLy9HcnIywsLCEBwcjFGjRlW7uWl2djZGjBiBwMBAhIeHY8aMGbDZbJ7clEaxePFi9OnTx3lSqMlkwsaNG53zfblvavLKK69AoVBgypQpzmm+3EfPP/88FAqFy6N79+7O+R7tGyGZlStXCo1GIz744ANx+PBhMWHCBBESEiLy8vK8XVqj27Bhg/j73/8uPv/8cwFArFmzxmX+K6+8IvR6vfjiiy/EDz/8IO644w4RHR0tLl686GwzbNgw0bdvX7F7927x3Xffic6dO4t7773Xw1vSOBISEsTSpUvFoUOHxMGDB8Vtt90moqKiRElJibPNxIkTRWRkpEhNTRX79+8XgwcPFkOGDHHOt9lsolevXiI+Pl4cOHBAbNiwQbRq1UrMmjXLG5vUoL788kuxfv16cfz4cZGZmSmeeeYZ4efnJw4dOiSE8O2++aO9e/eKDh06iD59+ognnnjCOd2X+2jOnDmiZ8+eIjc31/koKChwzvdk30gXXIMGDRLJycnO53a7XURERIiUlBQvVuV5fwwuh8MhjEajmD9/vnNaYWGh0Gq14t///rcQQogjR44IAGLfvn3ONhs3bhQKhUKcOXPGY7V7Sn5+vgAgtm3bJoSo6g8/Pz+xevVqZ5ujR48KACItLU0IUfXlQKlUCrPZ7GyzePFiodPphNVq9ewGeEDLli3Fe++9x775neLiYtGlSxexefNmceONNzqDy9f7aM6cOaJv3741zvN030i1q7CiogLp6emIj493TlMqlYiPj0daWpoXK/O+rKwsmM1ml77R6/WIi4tz9k1aWhpCQkIwYMAAZ5v4+HgolUrs2bPH4zU3NovFAuC3izKnp6ejsrLSpY+6d++OqKgolz7q3bs3DAaDs01CQgKKiopw+PBhD1bfuOx2O1auXInS0lKYTCb2ze8kJydjxIgRLn0B8P0DACdOnEBERAQ6duyIsWPHIjs7G4Dn+0aqi+yeO3cOdrvdZcMBwGAw4NixY16qqmkwm80AUGPfXJpnNpsRHh7uMl+tViM0NNTZprlwOByYMmUKrr32WvTq1QtA1fZrNBqEhIS4tP1jH9XUh5fmyS4jIwMmkwnl5eUIDg7GmjVrEBMTg4MHD/p83wDAypUr8f3332Pfvn3V5vn6+ycuLg7Lli1Dt27dkJubi7lz5+L666/HoUOHPN43UgUX0dVKTk7GoUOHsGPHDm+X0qR069YNBw8ehMViwX/+8x8kJiZi27Zt3i6rScjJycETTzyBzZs3w9/f39vlNDnDhw93/rtPnz6Ii4tD+/btsWrVKgQEBHi0Fql2FbZq1QoqlaraSJW8vDwYjUYvVdU0XNr+K/WN0WhEfn6+y3ybzYYLFy40q/6bNGkS1q1bh2+//Rbt2rVzTjcajaioqEBhYaFL+z/2UU19eGme7DQaDTp37ozY2FikpKSgb9++ePPNN9k3qNrdlZ+fj/79+0OtVkOtVmPbtm1YsGAB1Go1DAaDz/fR74WEhKBr1644efKkx98/UgWXRqNBbGwsUlNTndMcDgdSU1NhMpm8WJn3RUdHw2g0uvRNUVER9uzZ4+wbk8mEwsJCpKenO9ts2bIFDocDcXFxHq+5oQkhMGnSJKxZswZbtmxBdHS0y/zY2Fj4+fm59FFmZiays7Nd+igjI8Ml4Ddv3gydToeYmBjPbIgHORwOWK1W9g2AoUOHIiMjAwcPHnQ+BgwYgLFjxzr/7et99HslJSU4deoU2rRp4/n3j9tDS7xs5cqVQqvVimXLlokjR46IRx99VISEhLiMVGmuiouLxYEDB8SBAwcEAPHaa6+JAwcOiF9++UUIUTUcPiQkRKxdu1b8+OOP4s4776xxOPw111wj9uzZI3bs2CG6dOnSbIbDJyUlCb1eL7Zu3eoyZLesrMzZZuLEiSIqKkps2bJF7N+/X5hMJmEymZzzLw3ZvfXWW8XBgwfFpk2bROvWrZvFcOann35abNu2TWRlZYkff/xRPP3000KhUIivv/5aCOHbfXM5vx9VKIRv99H06dPF1q1bRVZWlti5c6eIj48XrVq1Evn5+UIIz/aNdMElhBBvvfWWiIqKEhqNRgwaNEjs3r3b2yV5xLfffisAVHskJiYKIaqGxD/77LPCYDAIrVYrhg4dKjIzM11e4/z58+Lee+8VwcHBQqfTiYceekgUFxd7YWsaXk19A0AsXbrU2ebixYvi8ccfFy1bthSBgYHiz3/+s8jNzXV5nZ9//lkMHz5cBAQEiFatWonp06eLyspKD29Nw3v44YdF+/bthUajEa1btxZDhw51hpYQvt03l/PH4PLlPhozZoxo06aN0Gg0om3btmLMmDHi5MmTzvme7Bve1oSIiKQi1TEuIiIiBhcREUmFwUVERFJhcBERkVQYXEREJBUGFxERSYXBRUREUmFwERGRVBhcREQkFQYXERFJhcFFRERSYXAREZFU/h+YPRkDNiEgcwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=False)\n",
    "valid_dataloader = DataLoader(valid_dataset, batch_size=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = next(iter(train_dataloader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pixel_values torch.Size([8, 3, 512, 512])\n",
      "labels torch.Size([8, 512, 512])\n"
     ]
    }
   ],
   "source": [
    "for k,v in batch.items():\n",
    "    print(k, v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 512, 512])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch[\"labels\"].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !git clone https://github.com/phiyodr/dacl10k-toolkit.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\vravula\\Downloads\\Cracks\\dacl10k-toolkit\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\vravula\\Downloads\\Cracks\\.venv\\Lib\\site-packages\\IPython\\core\\magics\\osm.py:417: UserWarning: using dhist requires you to install the `pickleshare` library.\n",
      "  self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n"
     ]
    }
   ],
   "source": [
    "%cd C:\\Users\\vravula\\Downloads\\Cracks\\dacl10k-toolkit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from dacl10k.dacl10kdataset import Dacl10kDataset\n",
    "\n",
    "# split = \"validation\" # \"train\", \"testdev\" (later also \"testchallenge\")\n",
    "# path_to_data = \"../data_megazord/a\"\n",
    "# dataset = Dacl10kDataset(split, path_to_data, resize_mask=(512, 512), resize_img=(512, 512), normalize_img=True)\n",
    "\n",
    "# img, mask = dataset[10]\n",
    "# img.shape, mask.shape\n",
    "\n",
    "# target_list = dataset.TARGET_LIST\n",
    "# target_list\n",
    "# #> (torch.Size([3, 512, 512]), torch.Size([19, 512, 512]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_list = ['Crack',\n",
    " 'ACrack',\n",
    " 'Wetspot',\n",
    " 'Efflorescence',\n",
    " 'Rust',\n",
    " 'Rockpocket',\n",
    " 'Hollowareas',\n",
    " 'Cavity',\n",
    " 'Spalling',\n",
    " 'Graffiti',\n",
    " 'Weathering',\n",
    " 'Restformwork',\n",
    " 'ExposedRebars',\n",
    " 'Bearing',\n",
    " 'EJoint',\n",
    " 'Drainage',\n",
    " 'PEquipment',\n",
    " 'JTape',\n",
    " 'WConccor']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 'Crack',\n",
       " 2: 'ACrack',\n",
       " 3: 'Wetspot',\n",
       " 4: 'Efflorescence',\n",
       " 5: 'Rust',\n",
       " 6: 'Rockpocket',\n",
       " 7: 'Hollowareas',\n",
       " 8: 'Cavity',\n",
       " 9: 'Spalling',\n",
       " 10: 'Graffiti',\n",
       " 11: 'Weathering',\n",
       " 12: 'Restformwork',\n",
       " 13: 'ExposedRebars',\n",
       " 14: 'Bearing',\n",
       " 15: 'EJoint',\n",
       " 16: 'Drainage',\n",
       " 17: 'PEquipment',\n",
       " 18: 'JTape',\n",
       " 19: 'WConccor',\n",
       " 0: 'unlabelled'}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dacl10k import utils\n",
    "\n",
    "id2label = dict(zip(range(1, len(utils.TARGET_LIST) + 1),utils.TARGET_LIST))\n",
    "id2label[0] = \"unlabelled\"\n",
    "id2label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "label2id = {v: k for k, v in id2label.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(id2label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of SegformerForSemanticSegmentation were not initialized from the model checkpoint at nvidia/mit-b5 and are newly initialized: ['decode_head.batch_norm.running_var', 'decode_head.batch_norm.weight', 'decode_head.batch_norm.num_batches_tracked', 'decode_head.linear_fuse.weight', 'decode_head.linear_c.1.proj.bias', 'decode_head.linear_c.0.proj.bias', 'decode_head.linear_c.2.proj.bias', 'decode_head.linear_c.3.proj.bias', 'decode_head.linear_c.2.proj.weight', 'decode_head.batch_norm.bias', 'decode_head.classifier.bias', 'decode_head.batch_norm.running_mean', 'decode_head.classifier.weight', 'decode_head.linear_c.0.proj.weight', 'decode_head.linear_c.3.proj.weight', 'decode_head.linear_c.1.proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = SegformerForSemanticSegmentation.from_pretrained(\"nvidia/mit-b5\", \n",
    "                                                         ignore_mismatched_sizes=True,\n",
    "                                                         num_labels=len(id2label), \n",
    "                                                         id2label=id2label, \n",
    "                                                         label2id=label2id,\n",
    "                                                         reshape_last_stage=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SegformerForSemanticSegmentation(\n",
      "  (segformer): SegformerModel(\n",
      "    (encoder): SegformerEncoder(\n",
      "      (patch_embeddings): ModuleList(\n",
      "        (0): SegformerOverlapPatchEmbeddings(\n",
      "          (proj): Conv2d(3, 64, kernel_size=(7, 7), stride=(4, 4), padding=(3, 3))\n",
      "          (layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (1): SegformerOverlapPatchEmbeddings(\n",
      "          (proj): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "          (layer_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (2): SegformerOverlapPatchEmbeddings(\n",
      "          (proj): Conv2d(128, 320, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "          (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "        (3): SegformerOverlapPatchEmbeddings(\n",
      "          (proj): Conv2d(320, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "          (layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "      )\n",
      "      (block): ModuleList(\n",
      "        (0): ModuleList(\n",
      "          (0): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (key): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (value): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(64, 64, kernel_size=(8, 8), stride=(8, 8))\n",
      "                (layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): Identity()\n",
      "            (layer_norm_2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=64, out_features=256, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=256, out_features=64, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (1): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (key): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (value): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(64, 64, kernel_size=(8, 8), stride=(8, 8))\n",
      "                (layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.0019607844296842813)\n",
      "            (layer_norm_2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=64, out_features=256, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=256, out_features=64, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (2): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (key): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (value): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(64, 64, kernel_size=(8, 8), stride=(8, 8))\n",
      "                (layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=64, out_features=64, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.003921568859368563)\n",
      "            (layer_norm_2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=64, out_features=256, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=256, out_features=64, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "        (1): ModuleList(\n",
      "          (0): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (key): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (value): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(128, 128, kernel_size=(4, 4), stride=(4, 4))\n",
      "                (layer_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.005882353521883488)\n",
      "            (layer_norm_2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=128, out_features=512, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=512, out_features=128, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (1): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (key): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (value): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(128, 128, kernel_size=(4, 4), stride=(4, 4))\n",
      "                (layer_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.007843137718737125)\n",
      "            (layer_norm_2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=128, out_features=512, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=512, out_features=128, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (2): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (key): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (value): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(128, 128, kernel_size=(4, 4), stride=(4, 4))\n",
      "                (layer_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.009803921915590763)\n",
      "            (layer_norm_2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=128, out_features=512, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=512, out_features=128, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (3): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (key): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (value): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(128, 128, kernel_size=(4, 4), stride=(4, 4))\n",
      "                (layer_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.011764707043766975)\n",
      "            (layer_norm_2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=128, out_features=512, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=512, out_features=128, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (4): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (key): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (value): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(128, 128, kernel_size=(4, 4), stride=(4, 4))\n",
      "                (layer_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.013725491240620613)\n",
      "            (layer_norm_2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=128, out_features=512, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=512, out_features=128, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (5): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (key): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (value): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(128, 128, kernel_size=(4, 4), stride=(4, 4))\n",
      "                (layer_norm): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=128, out_features=128, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.01568627543747425)\n",
      "            (layer_norm_2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=128, out_features=512, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=512, out_features=128, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "        (2): ModuleList(\n",
      "          (0): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.01764705963432789)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (1): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.019607843831181526)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (2): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.021568628028035164)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (3): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.02352941408753395)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (4): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.02549019828438759)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (5): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.027450982481241226)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (6): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.029411766678094864)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (7): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.0313725508749485)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (8): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.03333333507180214)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (9): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.03529411926865578)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (10): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.037254903465509415)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (11): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.03921568766236305)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (12): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.04117647185921669)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (13): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.04313725605607033)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (14): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.045098040252923965)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (15): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.0470588281750679)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (16): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.04901961237192154)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (17): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.05098038911819458)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (18): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.05294117331504822)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (19): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.054901961237192154)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (20): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.05686274543404579)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (21): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.05882352963089943)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (22): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.06078431382775307)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (23): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.0627450942993164)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (24): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.06470587849617004)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (25): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.06666666269302368)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (26): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.06862744688987732)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (27): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.07058823108673096)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (28): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.0725490152835846)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (29): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.07450979948043823)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (30): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.07647058367729187)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (31): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.0784313753247261)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (32): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.08039215952157974)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (33): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.08235294371843338)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (34): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.08431372791528702)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (35): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.08627451211214066)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (36): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.0882352963089943)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (37): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.09019608050584793)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (38): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.09215686470270157)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (39): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (key): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (value): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "                (sr): Conv2d(320, 320, kernel_size=(2, 2), stride=(2, 2))\n",
      "                (layer_norm): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=320, out_features=320, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.0941176488995552)\n",
      "            (layer_norm_2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=320, out_features=1280, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(1280, 1280, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1280)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=1280, out_features=320, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "        (3): ModuleList(\n",
      "          (0): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (key): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (value): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.09607843309640884)\n",
      "            (layer_norm_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=512, out_features=2048, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(2048, 2048, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2048)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=2048, out_features=512, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (1): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (key): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (value): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.09803921729326248)\n",
      "            (layer_norm_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=512, out_features=2048, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(2048, 2048, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2048)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=2048, out_features=512, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (2): SegformerLayer(\n",
      "            (layer_norm_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "            (attention): SegformerAttention(\n",
      "              (self): SegformerEfficientSelfAttention(\n",
      "                (query): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (key): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (value): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "              (output): SegformerSelfOutput(\n",
      "                (dense): Linear(in_features=512, out_features=512, bias=True)\n",
      "                (dropout): Dropout(p=0.0, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (drop_path): SegformerDropPath(p=0.10000000149011612)\n",
      "            (layer_norm_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): SegformerMixFFN(\n",
      "              (dense1): Linear(in_features=512, out_features=2048, bias=True)\n",
      "              (dwconv): SegformerDWConv(\n",
      "                (dwconv): Conv2d(2048, 2048, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=2048)\n",
      "              )\n",
      "              (intermediate_act_fn): GELUActivation()\n",
      "              (dense2): Linear(in_features=2048, out_features=512, bias=True)\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (layer_norm): ModuleList(\n",
      "        (0): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "        (1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
      "        (2): LayerNorm((320,), eps=1e-05, elementwise_affine=True)\n",
      "        (3): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (decode_head): SegformerDecodeHead(\n",
      "    (linear_c): ModuleList(\n",
      "      (0): SegformerMLP(\n",
      "        (proj): Linear(in_features=64, out_features=768, bias=True)\n",
      "      )\n",
      "      (1): SegformerMLP(\n",
      "        (proj): Linear(in_features=128, out_features=768, bias=True)\n",
      "      )\n",
      "      (2): SegformerMLP(\n",
      "        (proj): Linear(in_features=320, out_features=768, bias=True)\n",
      "      )\n",
      "      (3): SegformerMLP(\n",
      "        (proj): Linear(in_features=512, out_features=768, bias=True)\n",
      "      )\n",
      "    )\n",
      "    (linear_fuse): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "    (batch_norm): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    (activation): ReLU()\n",
      "    (dropout): Dropout(p=0.1, inplace=False)\n",
      "    (classifier): Conv2d(768, 20, kernel_size=(1, 1), stride=(1, 1))\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from torchsummary import summary\n",
    "# summary(model, (3, 512,512))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Initialized!\n"
     ]
    }
   ],
   "source": [
    "optimizer = torch.optim.AdamW(model.parameters(), lr=0.00006)\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "print(\"Model Initialized!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wed Dec  6 12:55:27 2023       \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 537.42                 Driver Version: 537.42       CUDA Version: 12.2     |\n",
      "|-----------------------------------------+----------------------+----------------------+\n",
      "| GPU  Name                     TCC/WDDM  | Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                                         |                      |               MIG M. |\n",
      "|=========================================+======================+======================|\n",
      "|   0  NVIDIA GeForce RTX 4090      WDDM  | 00000000:01:00.0  On |                  Off |\n",
      "| 30%   42C    P2              56W / 450W |   2934MiB / 24564MiB |      9%      Default |\n",
      "|                                         |                      |                  N/A |\n",
      "+-----------------------------------------+----------------------+----------------------+\n",
      "                                                                                         \n",
      "+---------------------------------------------------------------------------------------+\n",
      "| Processes:                                                                            |\n",
      "|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |\n",
      "|        ID   ID                                                             Usage      |\n",
      "|=======================================================================================|\n",
      "|    0   N/A  N/A      2276      C   ...0_x64__qbz5n2kfra8p0\\python3.11.exe    N/A      |\n",
      "|    0   N/A  N/A      3184    C+G   ...nt.CBS_cw5n1h2txyewy\\SearchHost.exe    N/A      |\n",
      "|    0   N/A  N/A     10472    C+G   ...2txyewy\\StartMenuExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A     11156    C+G   C:\\Windows\\System32\\Taskmgr.exe           N/A      |\n",
      "|    0   N/A  N/A     11768    C+G   ...5n1h2txyewy\\ShellExperienceHost.exe    N/A      |\n",
      "|    0   N/A  N/A     14940    C+G   C:\\Windows\\explorer.exe                   N/A      |\n",
      "|    0   N/A  N/A     18148    C+G   ...l\\Microsoft\\Teams\\current\\Teams.exe    N/A      |\n",
      "|    0   N/A  N/A     21268    C+G   ...siveControlPanel\\SystemSettings.exe    N/A      |\n",
      "|    0   N/A  N/A     21660    C+G   ...l\\Microsoft\\Teams\\current\\Teams.exe    N/A      |\n",
      "|    0   N/A  N/A     23192    C+G   ...am Files\\Microsoft VS Code\\Code.exe    N/A      |\n",
      "+---------------------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for idx, batch in enumerate(next(iter(train_dataloader))):\n",
    "#     print(batch)\n",
    "#     # get the inputs;\n",
    "#     pixel_values = batch[\"pixel_values\"].to(device)\n",
    "#     print(pixel_values)\n",
    "#     labels = batch[\"labels\"].to(device)\n",
    "#     print(labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\vravula\\AppData\\Local\\Temp\\ipykernel_2276\\1103646631.py:3: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",
      "  metric = load_metric(\"mean_iou\")\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from datasets import load_metric\n",
    "metric = load_metric(\"mean_iou\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2601/2601 [35:53<00:00,  1.21it/s, Batch=2600, Pixel-wise accuracy=0.758, Loss=0.826]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Pixel-wise accuracy: 0.758387089435617         Train Loss: 0.8263831330819205         Val Pixel-wise accuracy: 0.7422011344159236         Val Loss: 0.8113183124713559\n",
      "Epoch: 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2601/2601 [35:02<00:00,  1.24it/s, Batch=2600, Pixel-wise accuracy=0.832, Loss=0.522]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Pixel-wise accuracy: 0.8322478743527129         Train Loss: 0.5215625040596927         Val Pixel-wise accuracy: 0.6599329500250477         Val Loss: 1.0062710669278447\n",
      "Epoch: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2601/2601 [35:07<00:00,  1.23it/s, Batch=2600, Pixel-wise accuracy=0.877, Loss=0.376]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Pixel-wise accuracy: 0.87702280192685         Train Loss: 0.3761573561117402         Val Pixel-wise accuracy: 0.6032302757429946         Val Loss: 1.1691145026455811\n",
      "Epoch: 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2601/2601 [1:01:11<00:00,  1.41s/it, Batch=2600, Pixel-wise accuracy=0.904, Loss=0.29] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Pixel-wise accuracy: 0.904046184051408         Train Loss: 0.290230885949394         Val Pixel-wise accuracy: 0.7327296835477234         Val Loss: 0.9419796792942969\n",
      "Epoch: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2601/2601 [2:20:27<00:00,  3.24s/it, Batch=2600, Pixel-wise accuracy=0.918, Loss=0.244]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Pixel-wise accuracy: 0.9179160027172143         Train Loss: 0.2440228313307884         Val Pixel-wise accuracy: 0.7339042204976733         Val Loss: 0.9815369809015853\n",
      "Epoch: 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 5/2601 [00:20<2:55:26,  4.05s/it, Batch=5, Pixel-wise accuracy=0.886, Loss=0.32] \n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\vravula\\Downloads\\Cracks\\SegFormer_b3_50e_2aug.ipynb Cell 49\u001b[0m line \u001b[0;36m4\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/vravula/Downloads/Cracks/SegFormer_b3_50e_2aug.ipynb#X66sZmlsZQ%3D%3D?line=36'>37</a>\u001b[0m     pbar\u001b[39m.\u001b[39mset_postfix({\u001b[39m'\u001b[39m\u001b[39mBatch\u001b[39m\u001b[39m'\u001b[39m: idx, \u001b[39m'\u001b[39m\u001b[39mPixel-wise accuracy\u001b[39m\u001b[39m'\u001b[39m: \u001b[39msum\u001b[39m(accuracies)\u001b[39m/\u001b[39m\u001b[39mlen\u001b[39m(accuracies), \u001b[39m'\u001b[39m\u001b[39mLoss\u001b[39m\u001b[39m'\u001b[39m: \u001b[39msum\u001b[39m(losses)\u001b[39m/\u001b[39m\u001b[39mlen\u001b[39m(losses)})\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/vravula/Downloads/Cracks/SegFormer_b3_50e_2aug.ipynb#X66sZmlsZQ%3D%3D?line=38'>39</a>\u001b[0m     \u001b[39m# backward + optimize\u001b[39;00m\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/vravula/Downloads/Cracks/SegFormer_b3_50e_2aug.ipynb#X66sZmlsZQ%3D%3D?line=39'>40</a>\u001b[0m     loss\u001b[39m.\u001b[39;49mbackward()\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/vravula/Downloads/Cracks/SegFormer_b3_50e_2aug.ipynb#X66sZmlsZQ%3D%3D?line=40'>41</a>\u001b[0m     optimizer\u001b[39m.\u001b[39mstep()\n\u001b[0;32m     <a href='vscode-notebook-cell:/c%3A/Users/vravula/Downloads/Cracks/SegFormer_b3_50e_2aug.ipynb#X66sZmlsZQ%3D%3D?line=41'>42</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[1;32mc:\\Users\\vravula\\Downloads\\Cracks\\.venv\\Lib\\site-packages\\torch\\_tensor.py:492\u001b[0m, in \u001b[0;36mTensor.backward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    482\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[0;32m    483\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[0;32m    484\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[0;32m    485\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    490\u001b[0m         inputs\u001b[39m=\u001b[39minputs,\n\u001b[0;32m    491\u001b[0m     )\n\u001b[1;32m--> 492\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\n\u001b[0;32m    493\u001b[0m     \u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs\n\u001b[0;32m    494\u001b[0m )\n",
      "File \u001b[1;32mc:\\Users\\vravula\\Downloads\\Cracks\\.venv\\Lib\\site-packages\\torch\\autograd\\__init__.py:251\u001b[0m, in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    246\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[0;32m    248\u001b[0m \u001b[39m# The reason we repeat the same comment below is that\u001b[39;00m\n\u001b[0;32m    249\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[0;32m    250\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[1;32m--> 251\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[0;32m    252\u001b[0m     tensors,\n\u001b[0;32m    253\u001b[0m     grad_tensors_,\n\u001b[0;32m    254\u001b[0m     retain_graph,\n\u001b[0;32m    255\u001b[0m     create_graph,\n\u001b[0;32m    256\u001b[0m     inputs,\n\u001b[0;32m    257\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m    258\u001b[0m     accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m    259\u001b[0m )\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "best_val_accuracy = 0.0\n",
    "best_epoch = 0\n",
    "\n",
    "for epoch in range(1, 11):  # loop over the dataset multiple times\n",
    "    print(\"Epoch:\", epoch)\n",
    "    pbar = tqdm(train_dataloader)\n",
    "    accuracies = []\n",
    "    losses = []\n",
    "    val_accuracies = []\n",
    "    val_losses = []\n",
    "    model.train()\n",
    "    for idx, batch in enumerate(pbar):\n",
    "        # print(batch['pixel_values'].shape)\n",
    "        # print(batch['labels'].squeeze().unique())\n",
    "        \n",
    "        # get the inputs;\n",
    "        pixel_values = batch['pixel_values'].to(device)\n",
    "        labels = batch[\"labels\"].to(device)\n",
    "        \n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward\n",
    "        outputs = model(pixel_values=pixel_values, labels=labels)\n",
    "\n",
    "        # evaluate\n",
    "        upsampled_logits = nn.functional.interpolate(outputs.logits, size=labels.shape[-2:], mode=\"bilinear\", align_corners=False)\n",
    "        predicted = upsampled_logits.argmax(dim=1)\n",
    "        \n",
    "        mask = (labels != 255) # we don't include the background class in the accuracy calculation\n",
    "        pred_labels = predicted[mask].detach().cpu().numpy()\n",
    "        true_labels = labels[mask].detach().cpu().numpy()\n",
    "        accuracy = accuracy_score(pred_labels, true_labels)\n",
    "        loss = outputs.loss\n",
    "        accuracies.append(accuracy)\n",
    "        losses.append(loss.item())\n",
    "        pbar.set_postfix({'Batch': idx, 'Pixel-wise accuracy': sum(accuracies)/len(accuracies), 'Loss': sum(losses)/len(losses)})\n",
    "\n",
    "        # backward + optimize\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    else:\n",
    "        model.eval()\n",
    "        with torch.no_grad():\n",
    "            for idx, batch in enumerate(valid_dataloader):\n",
    "                pixel_values = batch[\"pixel_values\"].to(device)\n",
    "                labels = batch[\"labels\"].to(device)\n",
    "\n",
    "                outputs = model(pixel_values=pixel_values, labels=labels)\n",
    "                upsampled_logits = nn.functional.interpolate(outputs.logits, size=labels.shape[-2:], mode=\"bilinear\", align_corners=False)\n",
    "                predicted = upsampled_logits.argmax(dim=1)\n",
    "\n",
    "                # # note that the metric expects predictions + labels as numpy arrays\n",
    "                # metric.add_batch(predictions=predicted.detach().cpu().numpy(), references=labels.detach().cpu().numpy())\n",
    "\n",
    "                # # let's print loss and metrics every 100 batches\n",
    "                # if idx % 100 == 0:\n",
    "                #     metrics = metric._compute(\n",
    "                #         references = labels.cpu(),\n",
    "                #         predictions = predicted.cpu(),\n",
    "                #         num_labels=len(id2label), \n",
    "                #         ignore_index=255,\n",
    "                #         reduce_labels=False, # we've already reduced the labels before)\n",
    "                #     )\n",
    "\n",
    "                mask = (labels != 255) # we don't include the background class in the accuracy calculation\n",
    "                pred_labels = predicted[mask].detach().cpu().numpy()\n",
    "                true_labels = labels[mask].detach().cpu().numpy()\n",
    "                accuracy = accuracy_score(pred_labels, true_labels)\n",
    "                val_loss = outputs.loss\n",
    "                val_accuracies.append(accuracy)\n",
    "                val_losses.append(val_loss.item())\n",
    "\n",
    "    # Calculate average training and validation metrics\n",
    "    avg_train_accuracy = sum(accuracies) / len(accuracies)\n",
    "    avg_train_loss = sum(losses) / len(losses)\n",
    "    avg_val_accuracy = sum(val_accuracies) / len(val_accuracies)\n",
    "    avg_val_loss = sum(val_losses) / len(val_losses)\n",
    "\n",
    "    print(f\"Train Pixel-wise accuracy: {avg_train_accuracy}\\\n",
    "         Train Loss: {avg_train_loss}\\\n",
    "         Val Pixel-wise accuracy: {avg_val_accuracy}\\\n",
    "         Val Loss: {avg_val_loss}\")\n",
    "\n",
    "    # Check if the current model has the best validation accuracy\n",
    "    if avg_val_accuracy > best_val_accuracy:\n",
    "        best_val_accuracy = avg_val_accuracy\n",
    "        best_epoch = epoch\n",
    "\n",
    "        # Save the current best model\n",
    "        torch.save(model, f\"C:/Users/vravula/Downloads/Cracks/segformer_checkpoints/best_model_epoch_{epoch}.pt\")\n",
    "\n",
    "print(f\"Best model achieved at epoch {best_epoch} with validation accuracy: {best_val_accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model, f\"C:/Users/vravula/Downloads/Cracks/segformer_checkpoints/b5_model_epoch_{epoch}.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}