qrnn_imdb_hyperband.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Oajq_A9Wb1mQ",
    "outputId": "a7f6939a-42f3-432a-db56-3fba005da0ea"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive/\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive/')\n",
    "import sys\n",
    "import os\n",
    "prefix = '/content/gdrive/My Drive/'\n",
    "# modify \"customized_path_to_your_homework\" here to where you uploaded your homework\n",
    "customized_path_to_your_homework = 'IDLSProject-main'\n",
    "sys_path = os.path.join(prefix, customized_path_to_your_homework)\n",
    "sys.path.append(sys_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "O36-vTu0b4Vq",
    "outputId": "f8ea5416-815c-4d08-b623-d33d28826499"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/gdrive/My Drive/IDLSProject-main-2\n"
     ]
    }
   ],
   "source": [
    "%cd '/content/gdrive/My Drive/IDLSProject-main-2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "yBS_Cwn9bQOa"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "data_dir = './data/pytorch'\n",
    "with open(os.path.join(data_dir, 'word_dict_qrnn.pkl'), \"rb\") as f:\n",
    "    word_dict = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FMsjwRQ9KBwk",
    "outputId": "b76e367b-487a-483a-e765-de850ca234a6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 502) (20000, 502)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "train = pd.read_csv(os.path.join(data_dir, 'train_qrnn.csv'), header=None, names=None)\n",
    "test_sample = pd.read_csv(os.path.join(data_dir, 'test_qrnn.csv'), header=None, names=None)\n",
    "print(train.shape, test_sample.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sn509rQ2KFBX",
    "outputId": "2ed29b11-4999-42a0-e65e-118495c07956"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((30000, 502), (10000, 502), (10000, 502))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "test, val = train_test_split(test_sample, test_size=0.5)\n",
    "train.shape, test.shape, val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UkVwncNSKHsE",
    "outputId": "96eb5fed-ad4a-43eb-aada-4be526513957"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10000, 1])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.utils.data\n",
    "\n",
    "# Turn the input pandas dataframe into tensors\n",
    "train_y = torch.from_numpy(train[[0]].values).float()\n",
    "train_X = torch.from_numpy(train.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "train_ds = torch.utils.data.TensorDataset(train_X, train_y)\n",
    "# Build the dataloader\n",
    "train_dl = torch.utils.data.DataLoader(train_ds, batch_size=50)\n",
    "\n",
    "######val data\n",
    "# Turn the input pandas dataframe into tensors\n",
    "val_y = torch.from_numpy(val[[0]].values).float()\n",
    "val_X = torch.from_numpy(val.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "val_ds = torch.utils.data.TensorDataset(val_X, val_y)\n",
    "# Build the dataloader\n",
    "val_dl = torch.utils.data.DataLoader(val_ds, batch_size=50)\n",
    "\n",
    "\n",
    "#### Test data\n",
    "# Turn the input pandas dataframe into tensors\n",
    "test_y = torch.from_numpy(test[[0]].values).float()\n",
    "test_X = torch.from_numpy(test.drop([0, 1], axis=1).values).long()\n",
    "\n",
    "# Build the dataset\n",
    "test_ds = torch.utils.data.TensorDataset(test_X, test_y)\n",
    "# Build the dataloader\n",
    "test_dl = torch.utils.data.DataLoader(test_ds, batch_size=50)\n",
    "print(test_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "jDvo2PsZtLpB"
   },
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class QRNNLayer(nn.Module):\n",
    "    def __init__(self,batch_size,input_size,n_filters,kernel_size,embed_size,device,dropout):\n",
    "        super(QRNNLayer,self).__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.input_size = input_size\n",
    "        self.n_filters = n_filters\n",
    "        self.kernel_size = kernel_size\n",
    "        self.embed_size = embed_size\n",
    "        self.dropout = torch.nn.Dropout(dropout)\n",
    "        self.device = device\n",
    "        self.conv1 = torch.nn.Conv1d(self.input_size,self.n_filters,self.kernel_size)\n",
    "        self.conv2 = torch.nn.Conv1d(self.input_size,self.n_filters,self.kernel_size)\n",
    "        self.conv3 = torch.nn.Conv1d(self.input_size,self.n_filters,self.kernel_size)\n",
    "    \n",
    "    def forward(self,masked_input, h, c):\n",
    "        Z,F,O = self.masked_conv(masked_input)\n",
    "        h, c = self.pool(c,Z,F,O)\n",
    "        masked_input = h\n",
    "        return masked_input,h,c\n",
    "    \n",
    "    def masked_conv(self,x):\n",
    "        pad = torch.zeros([self.batch_size,1,self.input_size],device=self.device)\n",
    "        x = torch.cat([pad,x],1).permute(0,2,1)\n",
    "        Z = torch.tanh((self.conv1(x)))\n",
    "        F = torch.sigmoid((self.conv2(x)))\n",
    "        O = torch.sigmoid((self.conv3(x)))\n",
    "        one_mask = torch.ones_like(F,device=self.device) - F\n",
    "        F = 1 - self.dropout(one_mask)\n",
    "        return Z.permute(0,2,1), F.permute(0,2,1), O.permute(0,2,1)\n",
    "    \n",
    "    def pool(self, prev_c,Z,F,O):\n",
    "        c = torch.mul(F,prev_c) + torch.mul(1-F,Z)\n",
    "        h = torch.mul(O,c)\n",
    "        return h,c\n",
    "\n",
    "class QRNN(nn.Module):\n",
    "    def __init__(self,trial,vocab_size,embed_size,n_filters,kernel_size,batch_size,seq_len,layers,device,dropout):\n",
    "        super(QRNN,self).__init__()\n",
    "        dropout = trial.suggest_uniform(\"dropout\",0.1, 0.6)\n",
    "        self.num_layer = trial.suggest_int(\"num_layer\",1,4)\n",
    "        self.embed_size = trial.suggest_int(\"embed_size\",16,128)\n",
    "        self.n_filters = trial.suggest_int(\"n_filters\",50,200)\n",
    "        # self.kernel_size = trial.suggest_int(\"kernel_size\",2,5)\n",
    "        # self.embed_size = embed_size\n",
    "        # self.n_filters = n_filters\n",
    "        self.kernel_size = kernel_size\n",
    "        self.batch_size = batch_size\n",
    "        self.seq_len = seq_len\n",
    "        # self.num_layer = layers\n",
    "        self.device = device\n",
    "        self.embedding = torch.nn.Embedding(vocab_size, self.embed_size)\n",
    "        self.dense = torch.nn.Linear(self.seq_len*self.n_filters,1)\n",
    "        self.QRNN_layers = torch.nn.ModuleList([QRNNLayer(self.batch_size,self.embed_size if l==0 else self.n_filters,\n",
    "                                                         self.n_filters,self.kernel_size,self.embed_size,self.device,\n",
    "                                                         dropout,) for l in range(self.num_layer)])\n",
    "        \n",
    "        \n",
    "    def forward(self, x, target):\n",
    "        x = self.embedding(x)\n",
    "        h = torch.zeros([self.batch_size,self.seq_len,self.n_filters],device=self.device)\n",
    "        c = torch.zeros_like(h,device=self.device)\n",
    "        \n",
    "        masked_input = x\n",
    "        for l,layer in enumerate(self.QRNN_layers):\n",
    "            masked_input,h,c = layer(masked_input,h,c)\n",
    "        dense_input = h.reshape([self.batch_size,-1])\n",
    "        logits = self.dense(dense_input)\n",
    "        prediction = torch.sigmoid(logits)\n",
    "        target = target.view([-1,1])\n",
    "        correct_pred = torch.eq(torch.round(prediction).type(target.type()),target)\n",
    "        accuracy = torch.sum(correct_pred)\n",
    "        return prediction, accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "mT8XRhwXKPVg"
   },
   "outputs": [],
   "source": [
    "data_dir = './qrnn-imdb-hyperband-trails_20/' # The folder we will use for storing data\n",
    "if not os.path.exists(data_dir): # Make sure that the folder exists\n",
    "    os.makedirs(data_dir)\n",
    "\n",
    "filename = \"\"\n",
    "def write_to_csv(trail_num, epochs, train_loss, train_acc, val_loss, val_acc, time_train):\n",
    "    global filename\n",
    "    filename = \"./qrnn-imdb-hyperband-trails_20/\"+str(trail_num)+\".csv\"\n",
    "    epoch = [i for i in range(epochs)]\n",
    "    df_metrics = pd.DataFrame(list(zip(epoch, train_loss, train_acc, val_loss, val_acc, time_train)), columns =['Epoch', 'train_loss', 'train_acc', 'val_loss', 'val_acc', 'train_time'])\n",
    "    df_metrics.to_csv(filename, index=False)    \n",
    "    \n",
    "def append_to_csv(epochs, accuracy):\n",
    "    acc = [accuracy for i in range(epochs)]\n",
    "    df_csv = pd.read_csv(filename)\n",
    "    df_csv['Test_Accuracy']  = acc\n",
    "    df_csv.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IAuhnuf8xFNR",
    "outputId": "917669e7-4006-48cd-90bc-ef34f948defc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4998\n",
      "5000\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "epochs = 10\n",
    "n_filters = 64\n",
    "kernel_size = 2\n",
    "layers= 2\n",
    "learning_rate = 0.001\n",
    "print(len(word_dict))\n",
    "vocab_size = 5000\n",
    "print(vocab_size)\n",
    "embed_dims = 32\n",
    "seq_len = 500\n",
    "dropout = 0.3\n",
    "batch_size=50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gKatN9l8yHrX"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "id": "OasEM6YFKTBw"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import optuna\n",
    "from optuna.trial import TrialState\n",
    "import torch.optim as optim\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "\n",
    "# Define an objective function to be minimized.\n",
    "def objective(trial):\n",
    "    optimizer_name = trial.suggest_categorical(\"optimizer\", [\"Adam\", \"RMSprop\", \"SGD\"])\n",
    "    #epochs = trial.suggest_int(\"epochs\",5,15)\n",
    "    lr = trial.suggest_loguniform(\"lr\", 1e-5, 1e-0)\n",
    "    #momentum = trial.suggest_uniform(\"momentum\", 0.0, 1.0)\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    model = QRNN(trial,vocab_size, embed_dims, n_filters, kernel_size, batch_size, seq_len, layers, device, dropout).to(device)\n",
    "    trial.set_user_attr(key=\"best_model\", value=model)\n",
    "    optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=lr)\n",
    "    criterion = torch.nn.BCELoss()\n",
    "    counter = 0\n",
    "    QRNN_acc = []\n",
    "    QRNN_valacc = []\n",
    "    train_loss_epoch = []\n",
    "    train_acc_epoch = []\n",
    "    val_loss_epoch = []\n",
    "    val_acc_epoch = []\n",
    "    time_epoch = []\n",
    "    final_val_acc = 0\n",
    "    model.train()\n",
    "    for e in range(epochs):\n",
    "        start_time = time.time()\n",
    "        train_loss = []\n",
    "        train_acc = []\n",
    "        for inputs, labels in train_dl:\n",
    "            inputs, labels = inputs.cuda(), labels.cuda()\n",
    "            model.zero_grad()\n",
    "            logits, accuracy = model(inputs,labels)\n",
    "            loss = criterion(logits,labels.float())\n",
    "            loss.backward()\n",
    "            nn.utils.clip_grad_norm_(model.parameters(), 5)\n",
    "            optimizer.step()\n",
    "            train_loss.append(loss.item())\n",
    "            train_acc.append(accuracy.item()*100/batch_size)\n",
    "            if counter%100==0:\n",
    "                print(\"Epoch: {}/{}\".format(e,epochs),\n",
    "                             \"\\tIteration: {}\".format(counter),\n",
    "                             \"\\tTrain Loss: {:.3f}\".format(loss.item()),\n",
    "                             \"\\tTrain Accuracy: {:.2f}\".format(accuracy.item()*100/batch_size))\n",
    "                QRNN_acc.append(accuracy.item()*100/batch_size)\n",
    "            counter += 1\n",
    "        train_loss_epoch.append(np.round(np.mean(train_loss), 3))\n",
    "        train_acc_epoch.append(np.round(np.mean(train_acc), 3))\n",
    "        print(\"\\tTrain Loss: {:.3f}\".format(np.mean(train_loss)), \"\\tTrain Acc: {:.3f}\".format(np.mean(train_acc)))\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            model.eval()\n",
    "            val_acc = []\n",
    "            val_loss = []\n",
    "            for inputs, labels in val_dl:\n",
    "                inputs_val, labels_val = inputs.cuda(), labels.cuda()\n",
    "                logits_val,accuracy_val = model(inputs_val,labels_val)\n",
    "                loss_val = criterion(logits_val,labels_val.float())\n",
    "                val_acc.append(accuracy_val.item()*100/batch_size)\n",
    "                val_loss.append(loss_val.item())\n",
    "            print(\"Val Loss: {:.3f}\".format(np.mean(val_loss)), \"\\tVal Acc: {:.3f}\".format(np.mean(val_acc)))\n",
    "            val_loss_epoch.append(np.round(np.mean(val_loss), 3))\n",
    "            val_acc_epoch.append(np.round(np.mean(val_acc), 3))\n",
    "            QRNN_valacc.append(np.mean(val_acc))\n",
    "            model.train()\n",
    "            final_val_acc = np.mean(val_acc)\n",
    "            end_time = time.time()-start_time\n",
    "            print(\"Time to train epoch: {0} s\".format(end_time))\n",
    "            time_epoch.append(np.round(end_time, 3))\n",
    "    write_to_csv(trial.number, epochs, train_loss_epoch, train_acc_epoch, val_loss_epoch, val_acc_epoch, time_epoch)\n",
    "    return final_val_acc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "yM6NlQlNqA4h"
   },
   "outputs": [],
   "source": [
    "def callback(study, trial):\n",
    "    if study.best_trial.number == trial.number:\n",
    "        study.set_user_attr(key=\"best_model\", value=trial.user_attrs[\"best_model\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 409
    },
    "id": "QQ7svJFbqH5J",
    "outputId": "1d0f72f5-3712-4161-8c83-eb8e938fd588"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:04:49,527]\u001b[0m A new study created in memory with name: no-name-0e99f648-a883-4815-b53f-31623514f3e5\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.693 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 48.824 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 46.000 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 56.000 \tTrain Accuracy: 44.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.510 \tTrain Acc: 50.110\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.429874181747437 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 46.000 \tTrain Accuracy: 54.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.911 \tTrain Acc: 49.997\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.41899585723877 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.920 \tTrain Acc: 50.073\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.39556097984314 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.912 \tTrain Acc: 50.083\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.425739288330078 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.923 \tTrain Acc: 50.073\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.431697368621826 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.930 \tTrain Acc: 50.063\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.413528203964233 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.901 \tTrain Acc: 50.027\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.39626955986023 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 46.000 \tTrain Accuracy: 54.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.841 \tTrain Acc: 50.120\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.402182579040527 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.937 \tTrain Acc: 50.037\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.404900550842285 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.945 \tTrain Acc: 50.053\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:09:03,677]\u001b[0m Trial 0 finished with value: 50.24 and parameters: {'optimizer': 'Adam', 'lr': 0.21626720751825737, 'dropout': 0.49825009795037967, 'num_layer': 4, 'embed_size': 37, 'n_filters': 192}. Best is trial 0 with value: 50.24.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 25.411388397216797 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.691 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.767 \tTrain Acc: 50.110\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 14.600804328918457 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 14.587499618530273 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 14.593849182128906 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.22426986694336 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.28270697593689 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 14.762256622314453 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 14.610205888748169 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 14.607249975204468 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 14.61180305480957 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:11:31,183]\u001b[0m Trial 1 finished with value: 50.24 and parameters: {'optimizer': 'RMSprop', 'lr': 0.018410213262972947, 'dropout': 0.377322840240922, 'num_layer': 4, 'embed_size': 21, 'n_filters': 94}. Best is trial 0 with value: 50.24.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 14.606648683547974 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.691 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.698 \tTrain Accuracy: 44.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.687 \tTrain Accuracy: 56.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.691 \tTrain Accuracy: 56.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.693 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.692 \tTrain Accuracy: 60.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.813\n",
      "Val Loss: 0.692 \tVal Acc: 52.050\n",
      "Time to train epoch: 6.162529945373535 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.681 \tTrain Accuracy: 60.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.687 \tTrain Accuracy: 60.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.697 \tTrain Accuracy: 42.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.694 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.698 \tTrain Accuracy: 42.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.688 \tTrain Accuracy: 52.00\n",
      "\tTrain Loss: 0.691 \tTrain Acc: 52.393\n",
      "Val Loss: 0.689 \tVal Acc: 54.520\n",
      "Time to train epoch: 6.162964105606079 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.675 \tTrain Accuracy: 58.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.681 \tTrain Accuracy: 56.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.675 \tTrain Accuracy: 60.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.685 \tTrain Accuracy: 58.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.672 \tTrain Accuracy: 62.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.653 \tTrain Accuracy: 66.00\n",
      "\tTrain Loss: 0.683 \tTrain Acc: 56.520\n",
      "Val Loss: 0.675 \tVal Acc: 59.530\n",
      "Time to train epoch: 6.158778429031372 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.660 \tTrain Accuracy: 58.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.657 \tTrain Accuracy: 58.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.637 \tTrain Accuracy: 70.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.676 \tTrain Accuracy: 48.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.580 \tTrain Accuracy: 74.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.548 \tTrain Accuracy: 80.00\n",
      "\tTrain Loss: 0.638 \tTrain Acc: 64.997\n",
      "Val Loss: 0.592 \tVal Acc: 69.620\n",
      "Time to train epoch: 6.164578199386597 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.537 \tTrain Accuracy: 78.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.558 \tTrain Accuracy: 70.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.569 \tTrain Accuracy: 66.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.517 \tTrain Accuracy: 82.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.459 \tTrain Accuracy: 76.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.469 \tTrain Accuracy: 74.00\n",
      "\tTrain Loss: 0.539 \tTrain Acc: 73.340\n",
      "Val Loss: 0.513 \tVal Acc: 75.210\n",
      "Time to train epoch: 6.165582656860352 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.417 \tTrain Accuracy: 78.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.478 \tTrain Accuracy: 80.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.557 \tTrain Accuracy: 70.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.447 \tTrain Accuracy: 78.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.443 \tTrain Accuracy: 74.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.478 \tTrain Accuracy: 74.00\n",
      "\tTrain Loss: 0.487 \tTrain Acc: 76.577\n",
      "Val Loss: 0.482 \tVal Acc: 77.400\n",
      "Time to train epoch: 6.1579749584198 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.375 \tTrain Accuracy: 84.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.447 \tTrain Accuracy: 80.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.542 \tTrain Accuracy: 76.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.416 \tTrain Accuracy: 80.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.382 \tTrain Accuracy: 84.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.417 \tTrain Accuracy: 76.00\n",
      "\tTrain Loss: 0.457 \tTrain Acc: 78.937\n",
      "Val Loss: 0.459 \tVal Acc: 78.640\n",
      "Time to train epoch: 6.162452936172485 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.349 \tTrain Accuracy: 82.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.433 \tTrain Accuracy: 78.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.493 \tTrain Accuracy: 78.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.382 \tTrain Accuracy: 80.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.400 \tTrain Accuracy: 78.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.388 \tTrain Accuracy: 80.00\n",
      "\tTrain Loss: 0.433 \tTrain Acc: 80.107\n",
      "Val Loss: 0.441 \tVal Acc: 79.840\n",
      "Time to train epoch: 6.162709951400757 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.333 \tTrain Accuracy: 86.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.357 \tTrain Accuracy: 86.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.474 \tTrain Accuracy: 78.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.357 \tTrain Accuracy: 80.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.357 \tTrain Accuracy: 82.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.418 \tTrain Accuracy: 76.00\n",
      "\tTrain Loss: 0.412 \tTrain Acc: 81.350\n",
      "Val Loss: 0.425 \tVal Acc: 80.900\n",
      "Time to train epoch: 6.170976400375366 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.347 \tTrain Accuracy: 86.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.349 \tTrain Accuracy: 84.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.467 \tTrain Accuracy: 76.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.359 \tTrain Accuracy: 82.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.392 \tTrain Accuracy: 80.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.359 \tTrain Accuracy: 78.00\n",
      "\tTrain Loss: 0.395 \tTrain Acc: 82.440\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:12:32,848]\u001b[0m Trial 2 finished with value: 81.78 and parameters: {'optimizer': 'RMSprop', 'lr': 7.295887349092209e-05, 'dropout': 0.5152273448151032, 'num_layer': 3, 'embed_size': 112, 'n_filters': 62}. Best is trial 2 with value: 81.78.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.412 \tVal Acc: 81.780\n",
      "Time to train epoch: 6.1763999462127686 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.698 \tTrain Accuracy: 48.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.684 \tTrain Accuracy: 60.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.687 \tTrain Accuracy: 62.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.678 \tTrain Accuracy: 64.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.693 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.681 \tTrain Accuracy: 54.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 51.543\n",
      "Val Loss: 0.690 \tVal Acc: 53.510\n",
      "Time to train epoch: 5.146304130554199 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.686 \tTrain Accuracy: 58.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.681 \tTrain Accuracy: 58.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.681 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.681 \tTrain Accuracy: 60.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.687 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.680 \tTrain Accuracy: 58.00\n",
      "\tTrain Loss: 0.687 \tTrain Acc: 54.733\n",
      "Val Loss: 0.687 \tVal Acc: 56.020\n",
      "Time to train epoch: 5.147298336029053 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.681 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.680 \tTrain Accuracy: 56.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.669 \tTrain Accuracy: 54.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.677 \tTrain Accuracy: 66.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.678 \tTrain Accuracy: 66.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.663 \tTrain Accuracy: 70.00\n",
      "\tTrain Loss: 0.680 \tTrain Acc: 57.977\n",
      "Val Loss: 0.682 \tVal Acc: 57.960\n",
      "Time to train epoch: 5.146355390548706 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.668 \tTrain Accuracy: 62.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.665 \tTrain Accuracy: 70.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.655 \tTrain Accuracy: 64.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.669 \tTrain Accuracy: 62.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.663 \tTrain Accuracy: 68.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.650 \tTrain Accuracy: 74.00\n",
      "\tTrain Loss: 0.670 \tTrain Acc: 61.157\n",
      "Val Loss: 0.674 \tVal Acc: 59.960\n",
      "Time to train epoch: 5.1426544189453125 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.656 \tTrain Accuracy: 66.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.659 \tTrain Accuracy: 64.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.651 \tTrain Accuracy: 60.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.656 \tTrain Accuracy: 60.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.640 \tTrain Accuracy: 66.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.619 \tTrain Accuracy: 74.00\n",
      "\tTrain Loss: 0.656 \tTrain Acc: 64.187\n",
      "Val Loss: 0.661 \tVal Acc: 62.570\n",
      "Time to train epoch: 5.1394102573394775 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.641 \tTrain Accuracy: 72.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.643 \tTrain Accuracy: 66.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.622 \tTrain Accuracy: 70.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.639 \tTrain Accuracy: 70.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.619 \tTrain Accuracy: 74.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.597 \tTrain Accuracy: 76.00\n",
      "\tTrain Loss: 0.639 \tTrain Acc: 67.210\n",
      "Val Loss: 0.644 \tVal Acc: 65.430\n",
      "Time to train epoch: 5.134787321090698 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.623 \tTrain Accuracy: 70.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.635 \tTrain Accuracy: 64.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.601 \tTrain Accuracy: 74.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.626 \tTrain Accuracy: 72.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.587 \tTrain Accuracy: 72.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.584 \tTrain Accuracy: 84.00\n",
      "\tTrain Loss: 0.617 \tTrain Acc: 69.760\n",
      "Val Loss: 0.623 \tVal Acc: 68.040\n",
      "Time to train epoch: 5.138561725616455 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.601 \tTrain Accuracy: 78.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.615 \tTrain Accuracy: 68.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.579 \tTrain Accuracy: 76.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.578 \tTrain Accuracy: 76.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.579 \tTrain Accuracy: 74.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.551 \tTrain Accuracy: 84.00\n",
      "\tTrain Loss: 0.591 \tTrain Acc: 72.813\n",
      "Val Loss: 0.598 \tVal Acc: 70.390\n",
      "Time to train epoch: 5.1338112354278564 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.570 \tTrain Accuracy: 82.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.578 \tTrain Accuracy: 74.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.569 \tTrain Accuracy: 66.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.540 \tTrain Accuracy: 76.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.543 \tTrain Accuracy: 76.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.524 \tTrain Accuracy: 88.00\n",
      "\tTrain Loss: 0.563 \tTrain Acc: 74.667\n",
      "Val Loss: 0.573 \tVal Acc: 72.260\n",
      "Time to train epoch: 5.134133815765381 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.549 \tTrain Accuracy: 78.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.569 \tTrain Accuracy: 68.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.527 \tTrain Accuracy: 70.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.508 \tTrain Accuracy: 80.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.508 \tTrain Accuracy: 76.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.511 \tTrain Accuracy: 84.00\n",
      "\tTrain Loss: 0.536 \tTrain Acc: 76.350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:13:24,265]\u001b[0m Trial 3 finished with value: 73.71 and parameters: {'optimizer': 'Adam', 'lr': 2.021322730665194e-05, 'dropout': 0.4726907785166157, 'num_layer': 1, 'embed_size': 122, 'n_filters': 114}. Best is trial 2 with value: 81.78.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.549 \tVal Acc: 73.710\n",
      "Time to train epoch: 5.1365790367126465 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.693 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.506 \tTrain Accuracy: 72.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.394 \tTrain Accuracy: 82.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.350 \tTrain Accuracy: 84.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.250 \tTrain Accuracy: 92.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.243 \tTrain Accuracy: 88.00\n",
      "\tTrain Loss: 0.455 \tTrain Acc: 77.773\n",
      "Val Loss: 0.347 \tVal Acc: 85.520\n",
      "Time to train epoch: 18.209531545639038 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.218 \tTrain Accuracy: 94.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.319 \tTrain Accuracy: 90.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.331 \tTrain Accuracy: 88.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.279 \tTrain Accuracy: 88.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.250 \tTrain Accuracy: 88.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.152 \tTrain Accuracy: 92.00\n",
      "\tTrain Loss: 0.268 \tTrain Acc: 88.850\n",
      "Val Loss: 0.356 \tVal Acc: 86.340\n",
      "Time to train epoch: 18.207002878189087 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.128 \tTrain Accuracy: 92.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.147 \tTrain Accuracy: 92.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.181 \tTrain Accuracy: 92.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.225 \tTrain Accuracy: 88.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.154 \tTrain Accuracy: 92.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.056 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.185 \tTrain Acc: 92.520\n",
      "Val Loss: 0.434 \tVal Acc: 85.760\n",
      "Time to train epoch: 18.205191612243652 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.100 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.078 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.063 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.078 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.057 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.041 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.141 \tTrain Acc: 94.620\n",
      "Val Loss: 0.485 \tVal Acc: 85.430\n",
      "Time to train epoch: 18.213261127471924 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.107 \tTrain Accuracy: 94.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.146 \tTrain Accuracy: 96.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.111 \tTrain Accuracy: 96.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.124 \tTrain Accuracy: 94.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.032 \tTrain Accuracy: 100.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.112 \tTrain Accuracy: 96.00\n",
      "\tTrain Loss: 0.117 \tTrain Acc: 95.763\n",
      "Val Loss: 0.606 \tVal Acc: 86.180\n",
      "Time to train epoch: 18.82714056968689 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.032 \tTrain Accuracy: 100.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.105 \tTrain Accuracy: 96.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.124 \tTrain Accuracy: 94.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.148 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.088 \tTrain Acc: 96.693\n",
      "Val Loss: 0.763 \tVal Acc: 86.680\n",
      "Time to train epoch: 18.2960684299469 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.064 \tTrain Accuracy: 98.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.049 \tTrain Accuracy: 98.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.281 \tTrain Accuracy: 96.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.119 \tTrain Accuracy: 92.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.114 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.073 \tTrain Acc: 97.357\n",
      "Val Loss: 0.719 \tVal Acc: 86.100\n",
      "Time to train epoch: 18.212345361709595 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.055 \tTrain Accuracy: 98.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.007 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.142 \tTrain Accuracy: 92.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.094 \tTrain Accuracy: 98.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.130 \tTrain Accuracy: 96.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.060 \tTrain Acc: 97.890\n",
      "Val Loss: 0.818 \tVal Acc: 86.830\n",
      "Time to train epoch: 18.200401544570923 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.008 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.031 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.192 \tTrain Accuracy: 96.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.001 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.038 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.039 \tTrain Accuracy: 96.00\n",
      "\tTrain Loss: 0.053 \tTrain Acc: 98.213\n",
      "Val Loss: 0.869 \tVal Acc: 86.440\n",
      "Time to train epoch: 18.193490743637085 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.065 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.097 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.006 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.046 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.010 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.049 \tTrain Acc: 98.283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:16:27,055]\u001b[0m Trial 4 finished with value: 86.49 and parameters: {'optimizer': 'Adam', 'lr': 0.0022513003466663305, 'dropout': 0.32164100200933143, 'num_layer': 3, 'embed_size': 99, 'n_filters': 164}. Best is trial 4 with value: 86.49.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 1.111 \tVal Acc: 86.490\n",
      "Time to train epoch: 18.20523691177368 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.692 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.690 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.695 \tTrain Accuracy: 42.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.696 \tTrain Accuracy: 44.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.691 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.698 \tTrain Accuracy: 42.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.263\n",
      "Val Loss: 0.693 \tVal Acc: 49.640\n",
      "Time to train epoch: 10.941740989685059 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.692 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.694 \tTrain Accuracy: 54.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.691 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.692 \tTrain Accuracy: 60.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.692 \tTrain Accuracy: 56.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.691 \tTrain Accuracy: 58.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.290\n",
      "Val Loss: 0.693 \tVal Acc: 50.040\n",
      "Time to train epoch: 11.240547180175781 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.692 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.694 \tTrain Accuracy: 42.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.695 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.692 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.689 \tTrain Accuracy: 64.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.692 \tTrain Accuracy: 58.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 49.777\n",
      "Val Loss: 0.693 \tVal Acc: 50.310\n",
      "Time to train epoch: 11.250977993011475 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.694 \tTrain Accuracy: 56.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.695 \tTrain Accuracy: 42.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.696 \tTrain Accuracy: 48.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.686 \tTrain Accuracy: 70.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.693 \tTrain Accuracy: 46.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.697 \tTrain Accuracy: 40.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.273\n",
      "Val Loss: 0.693 \tVal Acc: 50.220\n",
      "Time to train epoch: 11.019816160202026 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.687 \tTrain Accuracy: 66.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.692 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.694 \tTrain Accuracy: 48.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.690 \tTrain Accuracy: 60.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.697 \tTrain Accuracy: 44.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.700 \tTrain Accuracy: 38.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.237\n",
      "Val Loss: 0.693 \tVal Acc: 50.430\n",
      "Time to train epoch: 10.894699096679688 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.693 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.694 \tTrain Accuracy: 48.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.692 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.689 \tTrain Accuracy: 60.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.690 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.701 \tTrain Accuracy: 36.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 49.717\n",
      "Val Loss: 0.693 \tVal Acc: 50.570\n",
      "Time to train epoch: 10.890333652496338 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.692 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.693 \tTrain Accuracy: 58.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.698 \tTrain Accuracy: 42.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.699 \tTrain Accuracy: 38.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.687 \tTrain Accuracy: 62.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.690 \tTrain Accuracy: 60.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.390\n",
      "Val Loss: 0.693 \tVal Acc: 50.580\n",
      "Time to train epoch: 10.915156602859497 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.688 \tTrain Accuracy: 58.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.691 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.692 \tTrain Accuracy: 56.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.691 \tTrain Accuracy: 56.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.690 \tTrain Accuracy: 56.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.693 \tTrain Accuracy: 42.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.007\n",
      "Val Loss: 0.693 \tVal Acc: 50.600\n",
      "Time to train epoch: 10.903172492980957 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.696 \tTrain Accuracy: 42.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.699 \tTrain Accuracy: 28.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.689 \tTrain Accuracy: 56.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.695 \tTrain Accuracy: 48.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.692 \tTrain Accuracy: 54.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.697 \tTrain Accuracy: 54.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.553\n",
      "Val Loss: 0.693 \tVal Acc: 50.750\n",
      "Time to train epoch: 10.884688138961792 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.694 \tTrain Accuracy: 54.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.694 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.690 \tTrain Accuracy: 54.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.694 \tTrain Accuracy: 46.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.690 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.694 \tTrain Accuracy: 48.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.893\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:18:16,909]\u001b[0m Trial 5 finished with value: 50.69 and parameters: {'optimizer': 'SGD', 'lr': 4.983843109533488e-05, 'dropout': 0.4347038567647393, 'num_layer': 3, 'embed_size': 51, 'n_filters': 109}. Best is trial 4 with value: 86.49.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.693 \tVal Acc: 50.690\n",
      "Time to train epoch: 10.8956458568573 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.690 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.683 \tTrain Accuracy: 60.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.665 \tTrain Accuracy: 56.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.664 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.557 \tTrain Accuracy: 68.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.483 \tTrain Accuracy: 78.00\n",
      "\tTrain Loss: 0.601 \tTrain Acc: 66.313\n",
      "Val Loss: 0.472 \tVal Acc: 78.070\n",
      "Time to train epoch: 4.795611619949341 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.327 \tTrain Accuracy: 90.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.460 \tTrain Accuracy: 80.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.430 \tTrain Accuracy: 80.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.325 \tTrain Accuracy: 82.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.383 \tTrain Accuracy: 82.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.346 \tTrain Accuracy: 88.00\n",
      "\tTrain Loss: 0.381 \tTrain Acc: 83.260\n",
      "Val Loss: 0.393 \tVal Acc: 82.540\n",
      "Time to train epoch: 5.111645936965942 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.194 \tTrain Accuracy: 96.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.354 \tTrain Accuracy: 84.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.346 \tTrain Accuracy: 86.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.210 \tTrain Accuracy: 94.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.287 \tTrain Accuracy: 88.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.230 \tTrain Accuracy: 90.00\n",
      "\tTrain Loss: 0.285 \tTrain Acc: 88.397\n",
      "Val Loss: 0.368 \tVal Acc: 84.060\n",
      "Time to train epoch: 5.123187065124512 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.143 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.250 \tTrain Accuracy: 88.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.277 \tTrain Accuracy: 86.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.146 \tTrain Accuracy: 96.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.216 \tTrain Accuracy: 90.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.132 \tTrain Accuracy: 96.00\n",
      "\tTrain Loss: 0.224 \tTrain Acc: 91.397\n",
      "Val Loss: 0.366 \tVal Acc: 84.610\n",
      "Time to train epoch: 5.125848054885864 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.116 \tTrain Accuracy: 98.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.196 \tTrain Accuracy: 92.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.244 \tTrain Accuracy: 92.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.128 \tTrain Accuracy: 96.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.194 \tTrain Accuracy: 92.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.092 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.178 \tTrain Acc: 93.690\n",
      "Val Loss: 0.374 \tVal Acc: 84.690\n",
      "Time to train epoch: 5.1234660148620605 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.084 \tTrain Accuracy: 98.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.138 \tTrain Accuracy: 96.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.216 \tTrain Accuracy: 92.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.088 \tTrain Accuracy: 98.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.160 \tTrain Accuracy: 90.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.068 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.142 \tTrain Acc: 95.340\n",
      "Val Loss: 0.387 \tVal Acc: 84.870\n",
      "Time to train epoch: 5.127394199371338 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.074 \tTrain Accuracy: 98.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.092 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.182 \tTrain Accuracy: 92.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.061 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.128 \tTrain Accuracy: 94.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.040 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.112 \tTrain Acc: 96.577\n",
      "Val Loss: 0.405 \tVal Acc: 84.890\n",
      "Time to train epoch: 5.0541064739227295 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.063 \tTrain Accuracy: 98.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.083 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.116 \tTrain Accuracy: 96.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.041 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.118 \tTrain Accuracy: 96.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.035 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.088 \tTrain Acc: 97.597\n",
      "Val Loss: 0.426 \tVal Acc: 85.070\n",
      "Time to train epoch: 4.765685319900513 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.033 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.055 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.092 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.030 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.090 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.040 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.068 \tTrain Acc: 98.430\n",
      "Val Loss: 0.449 \tVal Acc: 84.890\n",
      "Time to train epoch: 4.76263952255249 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.036 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.035 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.057 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.020 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.079 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.014 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.053 \tTrain Acc: 98.930\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:19:06,689]\u001b[0m Trial 6 finished with value: 84.9 and parameters: {'optimizer': 'Adam', 'lr': 0.0003679553709735557, 'dropout': 0.11884023383101214, 'num_layer': 1, 'embed_size': 59, 'n_filters': 116}. Best is trial 4 with value: 86.49.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.487 \tVal Acc: 84.900\n",
      "Time to train epoch: 4.773707628250122 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.691 \tTrain Accuracy: 62.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.695 \tTrain Accuracy: 42.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.694 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.697 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.691 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.685 \tTrain Accuracy: 58.00\n",
      "\tTrain Loss: 0.694 \tTrain Acc: 50.207\n",
      "Val Loss: 0.693 \tVal Acc: 50.560\n",
      "Time to train epoch: 3.4830353260040283 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.701 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.699 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.696 \tTrain Accuracy: 46.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.695 \tTrain Accuracy: 44.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.691 \tTrain Accuracy: 58.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.696 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.607\n",
      "Val Loss: 0.693 \tVal Acc: 50.630\n",
      "Time to train epoch: 3.4797635078430176 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.691 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.691 \tTrain Accuracy: 48.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.692 \tTrain Accuracy: 64.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.694 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.696 \tTrain Accuracy: 54.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.686 \tTrain Accuracy: 60.00\n",
      "\tTrain Loss: 0.694 \tTrain Acc: 49.993\n",
      "Val Loss: 0.693 \tVal Acc: 50.910\n",
      "Time to train epoch: 3.5473010540008545 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.689 \tTrain Accuracy: 54.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.691 \tTrain Accuracy: 48.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.695 \tTrain Accuracy: 42.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.700 \tTrain Accuracy: 46.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.695 \tTrain Accuracy: 48.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.693 \tTrain Accuracy: 56.00\n",
      "\tTrain Loss: 0.694 \tTrain Acc: 50.443\n",
      "Val Loss: 0.693 \tVal Acc: 51.080\n",
      "Time to train epoch: 3.690258502960205 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.690 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.698 \tTrain Accuracy: 46.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.687 \tTrain Accuracy: 62.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.708 \tTrain Accuracy: 34.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.691 \tTrain Accuracy: 54.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.694 \tTrain Accuracy: 42.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.760\n",
      "Val Loss: 0.693 \tVal Acc: 51.500\n",
      "Time to train epoch: 3.6891891956329346 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.692 \tTrain Accuracy: 56.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.689 \tTrain Accuracy: 56.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.693 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.687 \tTrain Accuracy: 54.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.687 \tTrain Accuracy: 56.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.695 \tTrain Accuracy: 52.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 51.660\n",
      "Val Loss: 0.693 \tVal Acc: 51.740\n",
      "Time to train epoch: 3.692742347717285 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.691 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.687 \tTrain Accuracy: 60.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.687 \tTrain Accuracy: 54.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.696 \tTrain Accuracy: 44.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.694 \tTrain Accuracy: 56.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.695 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 0.692 \tTrain Acc: 51.473\n",
      "Val Loss: 0.692 \tVal Acc: 51.860\n",
      "Time to train epoch: 3.5052096843719482 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.701 \tTrain Accuracy: 40.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.684 \tTrain Accuracy: 66.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.693 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.696 \tTrain Accuracy: 44.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.698 \tTrain Accuracy: 48.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.683 \tTrain Accuracy: 56.00\n",
      "\tTrain Loss: 0.692 \tTrain Acc: 51.403\n",
      "Val Loss: 0.692 \tVal Acc: 52.120\n",
      "Time to train epoch: 3.472879648208618 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.692 \tTrain Accuracy: 56.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.686 \tTrain Accuracy: 56.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.694 \tTrain Accuracy: 54.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.694 \tTrain Accuracy: 44.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.684 \tTrain Accuracy: 62.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.683 \tTrain Accuracy: 56.00\n",
      "\tTrain Loss: 0.692 \tTrain Acc: 51.337\n",
      "Val Loss: 0.692 \tVal Acc: 52.190\n",
      "Time to train epoch: 3.4730377197265625 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.694 \tTrain Accuracy: 48.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.688 \tTrain Accuracy: 62.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.676 \tTrain Accuracy: 64.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.691 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.688 \tTrain Accuracy: 60.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.689 \tTrain Accuracy: 56.00\n",
      "\tTrain Loss: 0.692 \tTrain Acc: 51.813\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:19:42,203]\u001b[0m Trial 7 finished with value: 52.49 and parameters: {'optimizer': 'SGD', 'lr': 5.6670477569093623e-05, 'dropout': 0.5339269933534412, 'num_layer': 1, 'embed_size': 68, 'n_filters': 100}. Best is trial 4 with value: 86.49.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.692 \tVal Acc: 52.490\n",
      "Time to train epoch: 3.466829299926758 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.686 \tTrain Accuracy: 66.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.696 \tTrain Accuracy: 48.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.686 \tTrain Accuracy: 66.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.684 \tTrain Accuracy: 64.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.697 \tTrain Accuracy: 46.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.692 \tTrain Accuracy: 52.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.720\n",
      "Val Loss: 0.693 \tVal Acc: 51.530\n",
      "Time to train epoch: 3.9677627086639404 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.682 \tTrain Accuracy: 70.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.691 \tTrain Accuracy: 48.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.690 \tTrain Accuracy: 56.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.688 \tTrain Accuracy: 60.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.695 \tTrain Accuracy: 42.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.693 \tTrain Accuracy: 64.00\n",
      "\tTrain Loss: 0.692 \tTrain Acc: 51.460\n",
      "Val Loss: 0.692 \tVal Acc: 51.570\n",
      "Time to train epoch: 3.9745187759399414 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.684 \tTrain Accuracy: 64.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.699 \tTrain Accuracy: 44.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.686 \tTrain Accuracy: 56.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.688 \tTrain Accuracy: 54.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.695 \tTrain Accuracy: 46.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.689 \tTrain Accuracy: 66.00\n",
      "\tTrain Loss: 0.692 \tTrain Acc: 51.863\n",
      "Val Loss: 0.692 \tVal Acc: 51.770\n",
      "Time to train epoch: 3.9750006198883057 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.683 \tTrain Accuracy: 62.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.686 \tTrain Accuracy: 56.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.684 \tTrain Accuracy: 64.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.692 \tTrain Accuracy: 60.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.692 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.689 \tTrain Accuracy: 54.00\n",
      "\tTrain Loss: 0.692 \tTrain Acc: 52.047\n",
      "Val Loss: 0.692 \tVal Acc: 51.950\n",
      "Time to train epoch: 3.97847318649292 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.686 \tTrain Accuracy: 62.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.689 \tTrain Accuracy: 58.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.685 \tTrain Accuracy: 62.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.690 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.699 \tTrain Accuracy: 38.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.684 \tTrain Accuracy: 62.00\n",
      "\tTrain Loss: 0.691 \tTrain Acc: 52.573\n",
      "Val Loss: 0.692 \tVal Acc: 52.330\n",
      "Time to train epoch: 3.9899966716766357 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.688 \tTrain Accuracy: 62.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.696 \tTrain Accuracy: 38.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.680 \tTrain Accuracy: 64.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.686 \tTrain Accuracy: 60.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.697 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.683 \tTrain Accuracy: 66.00\n",
      "\tTrain Loss: 0.691 \tTrain Acc: 52.890\n",
      "Val Loss: 0.692 \tVal Acc: 52.620\n",
      "Time to train epoch: 3.990119218826294 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.684 \tTrain Accuracy: 54.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.687 \tTrain Accuracy: 58.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.682 \tTrain Accuracy: 60.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.687 \tTrain Accuracy: 60.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.690 \tTrain Accuracy: 56.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.682 \tTrain Accuracy: 64.00\n",
      "\tTrain Loss: 0.691 \tTrain Acc: 53.377\n",
      "Val Loss: 0.691 \tVal Acc: 52.730\n",
      "Time to train epoch: 3.98722767829895 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.689 \tTrain Accuracy: 46.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.680 \tTrain Accuracy: 54.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.676 \tTrain Accuracy: 68.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.686 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.685 \tTrain Accuracy: 54.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.682 \tTrain Accuracy: 64.00\n",
      "\tTrain Loss: 0.690 \tTrain Acc: 53.127\n",
      "Val Loss: 0.691 \tVal Acc: 52.730\n",
      "Time to train epoch: 3.9867517948150635 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.681 \tTrain Accuracy: 60.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.688 \tTrain Accuracy: 44.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.680 \tTrain Accuracy: 62.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.689 \tTrain Accuracy: 56.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.691 \tTrain Accuracy: 44.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.680 \tTrain Accuracy: 64.00\n",
      "\tTrain Loss: 0.690 \tTrain Acc: 53.240\n",
      "Val Loss: 0.691 \tVal Acc: 52.720\n",
      "Time to train epoch: 3.9811105728149414 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.680 \tTrain Accuracy: 56.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.687 \tTrain Accuracy: 54.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.684 \tTrain Accuracy: 56.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.690 \tTrain Accuracy: 56.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.692 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.681 \tTrain Accuracy: 64.00\n",
      "\tTrain Loss: 0.690 \tTrain Acc: 53.187\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:20:22,021]\u001b[0m Trial 8 finished with value: 52.72 and parameters: {'optimizer': 'SGD', 'lr': 7.797785916810317e-05, 'dropout': 0.2503782721488116, 'num_layer': 1, 'embed_size': 32, 'n_filters': 139}. Best is trial 4 with value: 86.49.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.691 \tVal Acc: 52.720\n",
      "Time to train epoch: 3.9736950397491455 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.697 \tTrain Accuracy: 34.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.686 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.619 \tTrain Accuracy: 78.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.554 \tTrain Accuracy: 68.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.578 \tTrain Accuracy: 74.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.418 \tTrain Accuracy: 82.00\n",
      "\tTrain Loss: 0.566 \tTrain Acc: 68.463\n",
      "Val Loss: 0.436 \tVal Acc: 80.400\n",
      "Time to train epoch: 5.769313097000122 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.377 \tTrain Accuracy: 82.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.340 \tTrain Accuracy: 86.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.326 \tTrain Accuracy: 84.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.435 \tTrain Accuracy: 80.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.466 \tTrain Accuracy: 78.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.326 \tTrain Accuracy: 84.00\n",
      "\tTrain Loss: 0.390 \tTrain Acc: 82.700\n",
      "Val Loss: 0.374 \tVal Acc: 83.760\n",
      "Time to train epoch: 5.772778511047363 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.249 \tTrain Accuracy: 94.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.264 \tTrain Accuracy: 90.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.285 \tTrain Accuracy: 86.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.332 \tTrain Accuracy: 84.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.349 \tTrain Accuracy: 82.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.289 \tTrain Accuracy: 86.00\n",
      "\tTrain Loss: 0.326 \tTrain Acc: 86.267\n",
      "Val Loss: 0.343 \tVal Acc: 85.460\n",
      "Time to train epoch: 5.773757457733154 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.184 \tTrain Accuracy: 96.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.210 \tTrain Accuracy: 90.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.268 \tTrain Accuracy: 88.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.268 \tTrain Accuracy: 90.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.273 \tTrain Accuracy: 82.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.231 \tTrain Accuracy: 86.00\n",
      "\tTrain Loss: 0.286 \tTrain Acc: 88.000\n",
      "Val Loss: 0.327 \tVal Acc: 86.520\n",
      "Time to train epoch: 5.793883562088013 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.138 \tTrain Accuracy: 98.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.173 \tTrain Accuracy: 96.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.257 \tTrain Accuracy: 88.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.240 \tTrain Accuracy: 90.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.215 \tTrain Accuracy: 84.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.184 \tTrain Accuracy: 94.00\n",
      "\tTrain Loss: 0.256 \tTrain Acc: 89.697\n",
      "Val Loss: 0.318 \tVal Acc: 87.200\n",
      "Time to train epoch: 5.791677713394165 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.131 \tTrain Accuracy: 96.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.176 \tTrain Accuracy: 90.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.258 \tTrain Accuracy: 90.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.196 \tTrain Accuracy: 90.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.221 \tTrain Accuracy: 88.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.144 \tTrain Accuracy: 96.00\n",
      "\tTrain Loss: 0.233 \tTrain Acc: 90.683\n",
      "Val Loss: 0.315 \tVal Acc: 87.420\n",
      "Time to train epoch: 5.769249439239502 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.089 \tTrain Accuracy: 98.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.155 \tTrain Accuracy: 96.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.236 \tTrain Accuracy: 92.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.141 \tTrain Accuracy: 96.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.123 \tTrain Accuracy: 96.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.151 \tTrain Accuracy: 96.00\n",
      "\tTrain Loss: 0.210 \tTrain Acc: 91.570\n",
      "Val Loss: 0.319 \tVal Acc: 87.380\n",
      "Time to train epoch: 5.77676248550415 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.066 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.158 \tTrain Accuracy: 94.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.258 \tTrain Accuracy: 90.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.118 \tTrain Accuracy: 92.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.137 \tTrain Accuracy: 94.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.098 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.189 \tTrain Acc: 92.670\n",
      "Val Loss: 0.334 \tVal Acc: 87.410\n",
      "Time to train epoch: 5.7737321853637695 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.073 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.148 \tTrain Accuracy: 96.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.209 \tTrain Accuracy: 92.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.077 \tTrain Accuracy: 96.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.111 \tTrain Accuracy: 96.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.108 \tTrain Accuracy: 94.00\n",
      "\tTrain Loss: 0.171 \tTrain Acc: 93.403\n",
      "Val Loss: 0.342 \tVal Acc: 87.450\n",
      "Time to train epoch: 5.767888784408569 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.051 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.157 \tTrain Accuracy: 96.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.207 \tTrain Accuracy: 92.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.096 \tTrain Accuracy: 96.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.094 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.113 \tTrain Accuracy: 94.00\n",
      "\tTrain Loss: 0.156 \tTrain Acc: 93.977\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:21:19,802]\u001b[0m Trial 9 finished with value: 87.48 and parameters: {'optimizer': 'RMSprop', 'lr': 0.00034669386244841604, 'dropout': 0.18879994913780898, 'num_layer': 3, 'embed_size': 93, 'n_filters': 57}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.355 \tVal Acc: 87.480\n",
      "Time to train epoch: 5.774643421173096 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.694 \tTrain Accuracy: 48.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.754 \tTrain Acc: 50.083\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.172085285186768 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.157715320587158 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.177862167358398 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.177510738372803 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.170443534851074 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.177868127822876 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.174488067626953 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.165710687637329 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.170516014099121 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:22:11,651]\u001b[0m Trial 10 finished with value: 50.24 and parameters: {'optimizer': 'RMSprop', 'lr': 0.0029627579557365107, 'dropout': 0.1020166910720832, 'num_layer': 2, 'embed_size': 88, 'n_filters': 68}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 5.181175470352173 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.690 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 46.291 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.640 \tTrain Accuracy: 70.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.565 \tTrain Accuracy: 64.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.579 \tTrain Accuracy: 74.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.428 \tTrain Accuracy: 86.00\n",
      "\tTrain Loss: 11.208 \tTrain Acc: 69.083\n",
      "Val Loss: 0.381 \tVal Acc: 83.740\n",
      "Time to train epoch: 17.97727680206299 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.204 \tTrain Accuracy: 92.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.286 \tTrain Accuracy: 92.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.376 \tTrain Accuracy: 86.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.319 \tTrain Accuracy: 84.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.340 \tTrain Accuracy: 86.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.260 \tTrain Accuracy: 90.00\n",
      "\tTrain Loss: 0.305 \tTrain Acc: 87.220\n",
      "Val Loss: 0.350 \tVal Acc: 85.910\n",
      "Time to train epoch: 17.983825206756592 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.106 \tTrain Accuracy: 98.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.186 \tTrain Accuracy: 94.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.207 \tTrain Accuracy: 88.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.167 \tTrain Accuracy: 94.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.146 \tTrain Accuracy: 92.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.174 \tTrain Accuracy: 90.00\n",
      "\tTrain Loss: 0.194 \tTrain Acc: 92.343\n",
      "Val Loss: 0.407 \tVal Acc: 85.970\n",
      "Time to train epoch: 17.97399616241455 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.080 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.123 \tTrain Accuracy: 94.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.199 \tTrain Accuracy: 90.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.058 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.142 \tTrain Accuracy: 94.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.076 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.117 \tTrain Acc: 95.663\n",
      "Val Loss: 0.515 \tVal Acc: 85.820\n",
      "Time to train epoch: 17.94866967201233 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.025 \tTrain Accuracy: 100.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.128 \tTrain Accuracy: 94.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.114 \tTrain Accuracy: 98.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.051 \tTrain Accuracy: 98.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.048 \tTrain Accuracy: 98.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.072 \tTrain Acc: 97.290\n",
      "Val Loss: 0.675 \tVal Acc: 85.980\n",
      "Time to train epoch: 17.956111669540405 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.085 \tTrain Accuracy: 98.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.078 \tTrain Accuracy: 96.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.166 \tTrain Accuracy: 92.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.014 \tTrain Accuracy: 100.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.021 \tTrain Accuracy: 100.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.017 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.055 \tTrain Acc: 98.053\n",
      "Val Loss: 0.751 \tVal Acc: 86.070\n",
      "Time to train epoch: 17.954727172851562 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.017 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.008 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.021 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.074 \tTrain Accuracy: 98.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.029 \tTrain Accuracy: 98.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.043 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.046 \tTrain Acc: 98.297\n",
      "Val Loss: 0.794 \tVal Acc: 86.340\n",
      "Time to train epoch: 17.95521378517151 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.005 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.101 \tTrain Accuracy: 98.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.040 \tTrain Accuracy: 98.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.004 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.039 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.019 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.040 \tTrain Acc: 98.583\n",
      "Val Loss: 0.970 \tVal Acc: 86.110\n",
      "Time to train epoch: 17.956223249435425 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.029 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.088 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.001 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.014 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.029 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.035 \tTrain Acc: 98.773\n",
      "Val Loss: 0.969 \tVal Acc: 85.700\n",
      "Time to train epoch: 17.95311975479126 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.013 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.046 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.041 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.041 \tTrain Accuracy: 96.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.029 \tTrain Acc: 98.937\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:25:11,351]\u001b[0m Trial 11 finished with value: 86.57 and parameters: {'optimizer': 'RMSprop', 'lr': 0.0014349162493183254, 'dropout': 0.25708540918708767, 'num_layer': 3, 'embed_size': 93, 'n_filters': 165}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 1.184 \tVal Acc: 86.570\n",
      "Time to train epoch: 18.007300853729248 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.697 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.656 \tTrain Accuracy: 62.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.475 \tTrain Accuracy: 78.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.482 \tTrain Accuracy: 76.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.524 \tTrain Accuracy: 78.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.387 \tTrain Accuracy: 82.00\n",
      "\tTrain Loss: 0.601 \tTrain Acc: 74.637\n",
      "Val Loss: 0.404 \tVal Acc: 82.510\n",
      "Time to train epoch: 10.80094051361084 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.153 \tTrain Accuracy: 98.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.315 \tTrain Accuracy: 84.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.317 \tTrain Accuracy: 88.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.293 \tTrain Accuracy: 94.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.338 \tTrain Accuracy: 88.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.214 \tTrain Accuracy: 90.00\n",
      "\tTrain Loss: 0.304 \tTrain Acc: 87.183\n",
      "Val Loss: 0.358 \tVal Acc: 85.490\n",
      "Time to train epoch: 10.812060117721558 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.079 \tTrain Accuracy: 100.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.227 \tTrain Accuracy: 90.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.225 \tTrain Accuracy: 94.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.178 \tTrain Accuracy: 94.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.204 \tTrain Accuracy: 90.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.145 \tTrain Accuracy: 94.00\n",
      "\tTrain Loss: 0.206 \tTrain Acc: 91.947\n",
      "Val Loss: 0.377 \tVal Acc: 85.780\n",
      "Time to train epoch: 10.806662797927856 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.048 \tTrain Accuracy: 98.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.146 \tTrain Accuracy: 94.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.132 \tTrain Accuracy: 92.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.096 \tTrain Accuracy: 96.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.108 \tTrain Accuracy: 96.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.063 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.137 \tTrain Acc: 94.983\n",
      "Val Loss: 0.437 \tVal Acc: 85.460\n",
      "Time to train epoch: 10.813565492630005 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.022 \tTrain Accuracy: 100.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.124 \tTrain Accuracy: 92.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.098 \tTrain Accuracy: 98.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.040 \tTrain Accuracy: 98.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.057 \tTrain Accuracy: 100.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.035 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.085 \tTrain Acc: 97.200\n",
      "Val Loss: 0.505 \tVal Acc: 85.250\n",
      "Time to train epoch: 10.841090440750122 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.015 \tTrain Accuracy: 100.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.073 \tTrain Accuracy: 98.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.036 \tTrain Accuracy: 100.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.071 \tTrain Accuracy: 96.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.023 \tTrain Accuracy: 100.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.011 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.051 \tTrain Acc: 98.437\n",
      "Val Loss: 0.608 \tVal Acc: 85.520\n",
      "Time to train epoch: 11.238024473190308 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.034 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.047 \tTrain Accuracy: 98.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.020 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.026 \tTrain Accuracy: 100.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.012 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.033 \tTrain Acc: 99.033\n",
      "Val Loss: 0.718 \tVal Acc: 85.220\n",
      "Time to train epoch: 11.232816457748413 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.006 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.026 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.010 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.016 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.006 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.024 \tTrain Acc: 99.340\n",
      "Val Loss: 0.794 \tVal Acc: 85.250\n",
      "Time to train epoch: 11.241914749145508 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.021 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.021 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.009 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.016 \tTrain Acc: 99.553\n",
      "Val Loss: 0.982 \tVal Acc: 85.270\n",
      "Time to train epoch: 11.231727838516235 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.003 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.075 \tTrain Accuracy: 96.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.006 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.002 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.014 \tTrain Acc: 99.603\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:27:01,639]\u001b[0m Trial 12 finished with value: 85.37 and parameters: {'optimizer': 'RMSprop', 'lr': 0.0007372059308417752, 'dropout': 0.20926977776473743, 'num_layer': 2, 'embed_size': 84, 'n_filters': 154}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.990 \tVal Acc: 85.370\n",
      "Time to train epoch: 11.236740350723267 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.692 \tTrain Accuracy: 56.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.807 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.996893405914307 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.851815700531006 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.84939432144165 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.838052988052368 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.83451771736145 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.8448703289032 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.813673734664917 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.799792051315308 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.800836086273193 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:30:30,127]\u001b[0m Trial 13 finished with value: 50.24 and parameters: {'optimizer': 'RMSprop', 'lr': 0.023855206713963545, 'dropout': 0.21632637329994245, 'num_layer': 3, 'embed_size': 101, 'n_filters': 195}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 20.820133924484253 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.691 \tTrain Accuracy: 64.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.686 \tTrain Accuracy: 56.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.533 \tTrain Accuracy: 68.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.484 \tTrain Accuracy: 68.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.571 \tTrain Accuracy: 70.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.380 \tTrain Accuracy: 84.00\n",
      "\tTrain Loss: 0.561 \tTrain Acc: 70.420\n",
      "Val Loss: 0.421 \tVal Acc: 81.000\n",
      "Time to train epoch: 24.11527180671692 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.333 \tTrain Accuracy: 88.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.360 \tTrain Accuracy: 86.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.360 \tTrain Accuracy: 86.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.273 \tTrain Accuracy: 84.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.415 \tTrain Accuracy: 76.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.287 \tTrain Accuracy: 86.00\n",
      "\tTrain Loss: 0.368 \tTrain Acc: 84.067\n",
      "Val Loss: 0.358 \tVal Acc: 84.840\n",
      "Time to train epoch: 24.10503625869751 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.194 \tTrain Accuracy: 94.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.250 \tTrain Accuracy: 96.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.311 \tTrain Accuracy: 88.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.246 \tTrain Accuracy: 90.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.372 \tTrain Accuracy: 80.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.247 \tTrain Accuracy: 90.00\n",
      "\tTrain Loss: 0.296 \tTrain Acc: 88.003\n",
      "Val Loss: 0.340 \tVal Acc: 85.920\n",
      "Time to train epoch: 24.08920407295227 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.130 \tTrain Accuracy: 96.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.208 \tTrain Accuracy: 92.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.277 \tTrain Accuracy: 90.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.194 \tTrain Accuracy: 88.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.335 \tTrain Accuracy: 76.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.173 \tTrain Accuracy: 92.00\n",
      "\tTrain Loss: 0.241 \tTrain Acc: 90.347\n",
      "Val Loss: 0.342 \tVal Acc: 86.580\n",
      "Time to train epoch: 24.083552837371826 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.090 \tTrain Accuracy: 98.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.156 \tTrain Accuracy: 96.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.233 \tTrain Accuracy: 92.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.130 \tTrain Accuracy: 96.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.211 \tTrain Accuracy: 92.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.123 \tTrain Accuracy: 96.00\n",
      "\tTrain Loss: 0.196 \tTrain Acc: 92.407\n",
      "Val Loss: 0.367 \tVal Acc: 86.360\n",
      "Time to train epoch: 24.091623067855835 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.056 \tTrain Accuracy: 98.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.126 \tTrain Accuracy: 96.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.162 \tTrain Accuracy: 94.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.111 \tTrain Accuracy: 96.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.150 \tTrain Accuracy: 92.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.058 \tTrain Accuracy: 98.00\n",
      "\tTrain Loss: 0.153 \tTrain Acc: 94.033\n",
      "Val Loss: 0.448 \tVal Acc: 85.010\n",
      "Time to train epoch: 24.08596706390381 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.071 \tTrain Accuracy: 96.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.128 \tTrain Accuracy: 94.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.126 \tTrain Accuracy: 94.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.055 \tTrain Accuracy: 98.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.119 \tTrain Accuracy: 94.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.049 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.118 \tTrain Acc: 95.653\n",
      "Val Loss: 0.459 \tVal Acc: 86.070\n",
      "Time to train epoch: 24.077030181884766 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.014 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.079 \tTrain Accuracy: 96.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.063 \tTrain Accuracy: 98.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.026 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.031 \tTrain Accuracy: 100.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.028 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.094 \tTrain Acc: 96.513\n",
      "Val Loss: 0.583 \tVal Acc: 86.470\n",
      "Time to train epoch: 24.099225997924805 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.049 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.063 \tTrain Accuracy: 96.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.080 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.038 \tTrain Accuracy: 98.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.033 \tTrain Accuracy: 100.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.027 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.072 \tTrain Acc: 97.447\n",
      "Val Loss: 0.592 \tVal Acc: 86.130\n",
      "Time to train epoch: 24.076015949249268 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.024 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.036 \tTrain Accuracy: 100.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.083 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.042 \tTrain Accuracy: 96.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.044 \tTrain Accuracy: 98.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.018 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.056 \tTrain Acc: 98.007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:34:31,077]\u001b[0m Trial 14 finished with value: 85.87 and parameters: {'optimizer': 'RMSprop', 'lr': 0.0005478480799566137, 'dropout': 0.2906426217663813, 'num_layer': 4, 'embed_size': 81, 'n_filters': 167}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.663 \tVal Acc: 85.870\n",
      "Time to train epoch: 24.090371131896973 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.694 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.753 \tTrain Acc: 50.103\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.123028993606567 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.128709554672241 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.129626035690308 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.135562896728516 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.154428005218506 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.1488120555877686 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.144996166229248 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.134913444519043 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.129176378250122 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:35:12,451]\u001b[0m Trial 15 finished with value: 50.24 and parameters: {'optimizer': 'RMSprop', 'lr': 0.01350591513038033, 'dropout': 0.1732400999245856, 'num_layer': 2, 'embed_size': 102, 'n_filters': 51}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 4.114884376525879 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.692 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.805 \tTrain Acc: 50.003\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.443276405334473 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 48.933 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.873 \tTrain Acc: 50.113\n",
      "Val Loss: 49.752 \tVal Acc: 50.180\n",
      "Time to train epoch: 15.472093105316162 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 44.000 \tTrain Accuracy: 56.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.111 \tTrain Acc: 49.850\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.458806276321411 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.918 \tTrain Acc: 50.073\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.472048044204712 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.897 \tTrain Acc: 50.103\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.464871168136597 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.087\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.460236310958862 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.907 \tTrain Acc: 50.093\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.461910486221313 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.910 \tTrain Acc: 50.090\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.832087755203247 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.913 \tTrain Acc: 50.087\n",
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.605167865753174 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.900 \tTrain Acc: 50.100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:37:47,601]\u001b[0m Trial 16 finished with value: 50.24 and parameters: {'optimizer': 'RMSprop', 'lr': 0.2220378817182007, 'dropout': 0.5869182249117477, 'num_layer': 3, 'embed_size': 125, 'n_filters': 138}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 49.760 \tVal Acc: 50.240\n",
      "Time to train epoch: 15.445234537124634 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.693 \tTrain Accuracy: 56.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.688 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.669 \tTrain Accuracy: 68.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.677 \tTrain Accuracy: 58.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.512 \tTrain Accuracy: 78.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.423 \tTrain Accuracy: 86.00\n",
      "\tTrain Loss: 0.620 \tTrain Acc: 64.603\n",
      "Val Loss: 0.506 \tVal Acc: 75.110\n",
      "Time to train epoch: 6.440278053283691 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.401 \tTrain Accuracy: 84.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.526 \tTrain Accuracy: 72.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.471 \tTrain Accuracy: 74.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.446 \tTrain Accuracy: 82.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.433 \tTrain Accuracy: 74.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.349 \tTrain Accuracy: 84.00\n",
      "\tTrain Loss: 0.447 \tTrain Acc: 78.977\n",
      "Val Loss: 0.431 \tVal Acc: 80.280\n",
      "Time to train epoch: 6.4404261112213135 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.276 \tTrain Accuracy: 94.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.409 \tTrain Accuracy: 80.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.417 \tTrain Accuracy: 82.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.374 \tTrain Accuracy: 82.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.363 \tTrain Accuracy: 78.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.308 \tTrain Accuracy: 90.00\n",
      "\tTrain Loss: 0.375 \tTrain Acc: 83.447\n",
      "Val Loss: 0.393 \tVal Acc: 83.000\n",
      "Time to train epoch: 6.434900283813477 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.205 \tTrain Accuracy: 94.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.342 \tTrain Accuracy: 84.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.366 \tTrain Accuracy: 84.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.330 \tTrain Accuracy: 84.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.333 \tTrain Accuracy: 86.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.269 \tTrain Accuracy: 92.00\n",
      "\tTrain Loss: 0.328 \tTrain Acc: 86.183\n",
      "Val Loss: 0.372 \tVal Acc: 84.010\n",
      "Time to train epoch: 6.435118913650513 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.186 \tTrain Accuracy: 94.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.297 \tTrain Accuracy: 86.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.340 \tTrain Accuracy: 86.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.280 \tTrain Accuracy: 86.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.301 \tTrain Accuracy: 88.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.263 \tTrain Accuracy: 92.00\n",
      "\tTrain Loss: 0.293 \tTrain Acc: 87.920\n",
      "Val Loss: 0.357 \tVal Acc: 84.840\n",
      "Time to train epoch: 6.452174186706543 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.143 \tTrain Accuracy: 94.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.269 \tTrain Accuracy: 90.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.320 \tTrain Accuracy: 86.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.249 \tTrain Accuracy: 90.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.266 \tTrain Accuracy: 90.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.227 \tTrain Accuracy: 92.00\n",
      "\tTrain Loss: 0.264 \tTrain Acc: 89.353\n",
      "Val Loss: 0.348 \tVal Acc: 85.600\n",
      "Time to train epoch: 6.434015989303589 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.121 \tTrain Accuracy: 96.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.257 \tTrain Accuracy: 88.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.282 \tTrain Accuracy: 82.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.228 \tTrain Accuracy: 88.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.239 \tTrain Accuracy: 90.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.185 \tTrain Accuracy: 92.00\n",
      "\tTrain Loss: 0.239 \tTrain Acc: 90.380\n",
      "Val Loss: 0.342 \tVal Acc: 86.010\n",
      "Time to train epoch: 6.428474187850952 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.128 \tTrain Accuracy: 96.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.197 \tTrain Accuracy: 94.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.276 \tTrain Accuracy: 88.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.198 \tTrain Accuracy: 92.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.181 \tTrain Accuracy: 90.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.154 \tTrain Accuracy: 94.00\n",
      "\tTrain Loss: 0.216 \tTrain Acc: 91.417\n",
      "Val Loss: 0.342 \tVal Acc: 86.300\n",
      "Time to train epoch: 6.4286839962005615 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.106 \tTrain Accuracy: 96.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.185 \tTrain Accuracy: 94.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.234 \tTrain Accuracy: 88.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.172 \tTrain Accuracy: 94.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.161 \tTrain Accuracy: 90.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.146 \tTrain Accuracy: 96.00\n",
      "\tTrain Loss: 0.196 \tTrain Acc: 92.420\n",
      "Val Loss: 0.343 \tVal Acc: 86.280\n",
      "Time to train epoch: 6.423932790756226 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.096 \tTrain Accuracy: 96.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.194 \tTrain Accuracy: 90.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.207 \tTrain Accuracy: 90.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.184 \tTrain Accuracy: 92.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.122 \tTrain Accuracy: 94.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.094 \tTrain Accuracy: 100.00\n",
      "\tTrain Loss: 0.177 \tTrain Acc: 93.193\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:38:51,984]\u001b[0m Trial 17 finished with value: 86.39 and parameters: {'optimizer': 'RMSprop', 'lr': 0.00023452762646168778, 'dropout': 0.17037118533531562, 'num_layer': 2, 'embed_size': 73, 'n_filters': 78}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.348 \tVal Acc: 86.390\n",
      "Time to train epoch: 6.434581995010376 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.693 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 48.000 \tTrain Accuracy: 52.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 60.000 \tTrain Accuracy: 40.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 49.668 \tTrain Acc: 50.180\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 24.9097638130188 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 24.90338945388794 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 24.975793600082397 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 24.971235990524292 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 24.896645307540894 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 25.347522258758545 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 25.544517755508423 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 25.00546169281006 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n",
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 24.90600061416626 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 52.000 \tTrain Accuracy: 48.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 40.000 \tTrain Accuracy: 60.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 50.000 \tTrain Accuracy: 50.00\n",
      "\tTrain Loss: 50.090 \tTrain Acc: 49.910\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:43:02,443]\u001b[0m Trial 18 finished with value: 49.76 and parameters: {'optimizer': 'RMSprop', 'lr': 0.006047262814334825, 'dropout': 0.3720174176995784, 'num_layer': 4, 'embed_size': 90, 'n_filters': 176}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 50.240 \tVal Acc: 49.760\n",
      "Time to train epoch: 24.96263289451599 s\n",
      "Epoch: 0/10 \tIteration: 0 \tTrain Loss: 0.690 \tTrain Accuracy: 54.00\n",
      "Epoch: 0/10 \tIteration: 100 \tTrain Loss: 0.696 \tTrain Accuracy: 42.00\n",
      "Epoch: 0/10 \tIteration: 200 \tTrain Loss: 0.694 \tTrain Accuracy: 46.00\n",
      "Epoch: 0/10 \tIteration: 300 \tTrain Loss: 0.695 \tTrain Accuracy: 44.00\n",
      "Epoch: 0/10 \tIteration: 400 \tTrain Loss: 0.691 \tTrain Accuracy: 62.00\n",
      "Epoch: 0/10 \tIteration: 500 \tTrain Loss: 0.693 \tTrain Accuracy: 56.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.170\n",
      "Val Loss: 0.693 \tVal Acc: 49.690\n",
      "Time to train epoch: 15.040668725967407 s\n",
      "Epoch: 1/10 \tIteration: 600 \tTrain Loss: 0.695 \tTrain Accuracy: 44.00\n",
      "Epoch: 1/10 \tIteration: 700 \tTrain Loss: 0.695 \tTrain Accuracy: 44.00\n",
      "Epoch: 1/10 \tIteration: 800 \tTrain Loss: 0.696 \tTrain Accuracy: 42.00\n",
      "Epoch: 1/10 \tIteration: 900 \tTrain Loss: 0.694 \tTrain Accuracy: 38.00\n",
      "Epoch: 1/10 \tIteration: 1000 \tTrain Loss: 0.693 \tTrain Accuracy: 50.00\n",
      "Epoch: 1/10 \tIteration: 1100 \tTrain Loss: 0.691 \tTrain Accuracy: 58.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 49.867\n",
      "Val Loss: 0.693 \tVal Acc: 49.830\n",
      "Time to train epoch: 15.029553413391113 s\n",
      "Epoch: 2/10 \tIteration: 1200 \tTrain Loss: 0.696 \tTrain Accuracy: 46.00\n",
      "Epoch: 2/10 \tIteration: 1300 \tTrain Loss: 0.694 \tTrain Accuracy: 52.00\n",
      "Epoch: 2/10 \tIteration: 1400 \tTrain Loss: 0.695 \tTrain Accuracy: 44.00\n",
      "Epoch: 2/10 \tIteration: 1500 \tTrain Loss: 0.696 \tTrain Accuracy: 38.00\n",
      "Epoch: 2/10 \tIteration: 1600 \tTrain Loss: 0.693 \tTrain Accuracy: 50.00\n",
      "Epoch: 2/10 \tIteration: 1700 \tTrain Loss: 0.693 \tTrain Accuracy: 52.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 49.860\n",
      "Val Loss: 0.693 \tVal Acc: 49.700\n",
      "Time to train epoch: 15.034101724624634 s\n",
      "Epoch: 3/10 \tIteration: 1800 \tTrain Loss: 0.694 \tTrain Accuracy: 50.00\n",
      "Epoch: 3/10 \tIteration: 1900 \tTrain Loss: 0.694 \tTrain Accuracy: 46.00\n",
      "Epoch: 3/10 \tIteration: 2000 \tTrain Loss: 0.695 \tTrain Accuracy: 54.00\n",
      "Epoch: 3/10 \tIteration: 2100 \tTrain Loss: 0.691 \tTrain Accuracy: 54.00\n",
      "Epoch: 3/10 \tIteration: 2200 \tTrain Loss: 0.692 \tTrain Accuracy: 54.00\n",
      "Epoch: 3/10 \tIteration: 2300 \tTrain Loss: 0.693 \tTrain Accuracy: 46.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 49.963\n",
      "Val Loss: 0.693 \tVal Acc: 49.390\n",
      "Time to train epoch: 15.031807661056519 s\n",
      "Epoch: 4/10 \tIteration: 2400 \tTrain Loss: 0.694 \tTrain Accuracy: 48.00\n",
      "Epoch: 4/10 \tIteration: 2500 \tTrain Loss: 0.696 \tTrain Accuracy: 44.00\n",
      "Epoch: 4/10 \tIteration: 2600 \tTrain Loss: 0.693 \tTrain Accuracy: 42.00\n",
      "Epoch: 4/10 \tIteration: 2700 \tTrain Loss: 0.696 \tTrain Accuracy: 42.00\n",
      "Epoch: 4/10 \tIteration: 2800 \tTrain Loss: 0.692 \tTrain Accuracy: 54.00\n",
      "Epoch: 4/10 \tIteration: 2900 \tTrain Loss: 0.696 \tTrain Accuracy: 44.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.233\n",
      "Val Loss: 0.693 \tVal Acc: 49.370\n",
      "Time to train epoch: 15.032425165176392 s\n",
      "Epoch: 5/10 \tIteration: 3000 \tTrain Loss: 0.692 \tTrain Accuracy: 50.00\n",
      "Epoch: 5/10 \tIteration: 3100 \tTrain Loss: 0.694 \tTrain Accuracy: 44.00\n",
      "Epoch: 5/10 \tIteration: 3200 \tTrain Loss: 0.692 \tTrain Accuracy: 54.00\n",
      "Epoch: 5/10 \tIteration: 3300 \tTrain Loss: 0.693 \tTrain Accuracy: 48.00\n",
      "Epoch: 5/10 \tIteration: 3400 \tTrain Loss: 0.693 \tTrain Accuracy: 48.00\n",
      "Epoch: 5/10 \tIteration: 3500 \tTrain Loss: 0.692 \tTrain Accuracy: 46.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.070\n",
      "Val Loss: 0.693 \tVal Acc: 49.590\n",
      "Time to train epoch: 15.027495622634888 s\n",
      "Epoch: 6/10 \tIteration: 3600 \tTrain Loss: 0.693 \tTrain Accuracy: 54.00\n",
      "Epoch: 6/10 \tIteration: 3700 \tTrain Loss: 0.696 \tTrain Accuracy: 40.00\n",
      "Epoch: 6/10 \tIteration: 3800 \tTrain Loss: 0.697 \tTrain Accuracy: 42.00\n",
      "Epoch: 6/10 \tIteration: 3900 \tTrain Loss: 0.693 \tTrain Accuracy: 52.00\n",
      "Epoch: 6/10 \tIteration: 4000 \tTrain Loss: 0.692 \tTrain Accuracy: 50.00\n",
      "Epoch: 6/10 \tIteration: 4100 \tTrain Loss: 0.693 \tTrain Accuracy: 56.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 49.610\n",
      "Val Loss: 0.693 \tVal Acc: 49.450\n",
      "Time to train epoch: 15.0310537815094 s\n",
      "Epoch: 7/10 \tIteration: 4200 \tTrain Loss: 0.694 \tTrain Accuracy: 46.00\n",
      "Epoch: 7/10 \tIteration: 4300 \tTrain Loss: 0.694 \tTrain Accuracy: 48.00\n",
      "Epoch: 7/10 \tIteration: 4400 \tTrain Loss: 0.694 \tTrain Accuracy: 46.00\n",
      "Epoch: 7/10 \tIteration: 4500 \tTrain Loss: 0.693 \tTrain Accuracy: 48.00\n",
      "Epoch: 7/10 \tIteration: 4600 \tTrain Loss: 0.688 \tTrain Accuracy: 62.00\n",
      "Epoch: 7/10 \tIteration: 4700 \tTrain Loss: 0.694 \tTrain Accuracy: 46.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 49.553\n",
      "Val Loss: 0.693 \tVal Acc: 49.570\n",
      "Time to train epoch: 15.032242059707642 s\n",
      "Epoch: 8/10 \tIteration: 4800 \tTrain Loss: 0.690 \tTrain Accuracy: 58.00\n",
      "Epoch: 8/10 \tIteration: 4900 \tTrain Loss: 0.692 \tTrain Accuracy: 50.00\n",
      "Epoch: 8/10 \tIteration: 5000 \tTrain Loss: 0.695 \tTrain Accuracy: 44.00\n",
      "Epoch: 8/10 \tIteration: 5100 \tTrain Loss: 0.695 \tTrain Accuracy: 46.00\n",
      "Epoch: 8/10 \tIteration: 5200 \tTrain Loss: 0.694 \tTrain Accuracy: 42.00\n",
      "Epoch: 8/10 \tIteration: 5300 \tTrain Loss: 0.694 \tTrain Accuracy: 56.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 50.473\n",
      "Val Loss: 0.693 \tVal Acc: 49.690\n",
      "Time to train epoch: 15.179923057556152 s\n",
      "Epoch: 9/10 \tIteration: 5400 \tTrain Loss: 0.697 \tTrain Accuracy: 40.00\n",
      "Epoch: 9/10 \tIteration: 5500 \tTrain Loss: 0.695 \tTrain Accuracy: 42.00\n",
      "Epoch: 9/10 \tIteration: 5600 \tTrain Loss: 0.693 \tTrain Accuracy: 50.00\n",
      "Epoch: 9/10 \tIteration: 5700 \tTrain Loss: 0.689 \tTrain Accuracy: 68.00\n",
      "Epoch: 9/10 \tIteration: 5800 \tTrain Loss: 0.691 \tTrain Accuracy: 54.00\n",
      "Epoch: 9/10 \tIteration: 5900 \tTrain Loss: 0.692 \tTrain Accuracy: 52.00\n",
      "\tTrain Loss: 0.693 \tTrain Acc: 49.467\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m[I 2022-05-14 16:45:33,024]\u001b[0m Trial 19 finished with value: 49.66 and parameters: {'optimizer': 'SGD', 'lr': 1.0166476591232525e-05, 'dropout': 0.2818696043093479, 'num_layer': 3, 'embed_size': 111, 'n_filters': 138}. Best is trial 9 with value: 87.48.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.693 \tVal Acc: 49.660\n",
      "Time to train epoch: 15.105906248092651 s\n",
      "Study statistics: \n",
      "  Number of finished trials:  20\n",
      "  Number of complete trials:  20\n",
      "Best trial:\n",
      "  Value:  87.48\n",
      "  Params: \n",
      "    optimizer: RMSprop\n",
      "    lr: 0.00034669386244841604\n",
      "    dropout: 0.18879994913780898\n",
      "    num_layer: 3\n",
      "    embed_size: 93\n",
      "    n_filters: 57\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "from optuna.trial import TrialState\n",
    "study = optuna.create_study(direction=\"maximize\",pruner=optuna.pruners.HyperbandPruner(\n",
    "        min_resource=1, max_resource=epochs, reduction_factor=3\n",
    "    ),)\n",
    "study.optimize(objective, n_trials=20,callbacks=[callback])\n",
    "complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])\n",
    "print(\"Study statistics: \")\n",
    "print(\"  Number of finished trials: \", len(study.trials))\n",
    "print(\"  Number of complete trials: \", len(complete_trials))\n",
    "print(\"Best trial:\")\n",
    "trial = study.best_trial\n",
    "best_model=study.user_attrs[\"best_model\"]\n",
    "print(\"  Value: \", trial.value)\n",
    "print(\"  Params: \")\n",
    "for key, value in trial.params.items():\n",
    "    print(\"    {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "Njic3VLTwSPU"
   },
   "outputs": [],
   "source": [
    "data_dir = './qrnn-imdb-hyperband/' # The folder we will use for storing data\n",
    "if not os.path.exists(data_dir): # Make sure that the folder exists\n",
    "    os.makedirs(data_dir)\n",
    "    \n",
    "torch.save(best_model.state_dict(),\"./qrnn-imdb-hyperband/qrnn-imdb-hyperband_20.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "2D2JfLBeKjQ0"
   },
   "outputs": [],
   "source": [
    "criterion = torch.nn.BCELoss()\n",
    "def test(model, test_dl, epochs):\n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        test_acc = []\n",
    "        test_loss = []\n",
    "        for inputs, labels in test_dl:\n",
    "            input_test, labels_test = inputs.cuda(), labels.cuda()\n",
    "            logits_test, accuracy_test = model(input_test, labels_test)\n",
    "            loss_test = criterion(logits_test, labels_test.float())\n",
    "            test_acc.append(accuracy_test.item()*100/batch_size)\n",
    "            test_loss.append(loss_test.item())\n",
    "        print(\"Test Loss: {:.3f}\".format(np.mean(test_loss)), \"\\tTest Acc: {:.3f}\".format(np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "vR_FTuLSby1Y"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 0.325 \tTest Acc: 88.030\n"
     ]
    }
   ],
   "source": [
    "test(best_model, test_dl, epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "qrnn-imdb-hyperband.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "my_env2_gpu",
   "language": "python",
   "name": "my_env2_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}