Gradient_Steering.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Gradient Steering.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "mount_file_id": "1HZui3iBsp_ImABbDpwsugkNChgKIU1XM",
      "authorship_tag": "ABX9TyMORhtDEGQtBN5I3jJmKlY3",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kpi6research/Text-Style-Transfer-with-Gradient-Steering/blob/master/Gradient_Steering.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ADcPezFaI_J",
        "colab_type": "text"
      },
      "source": [
        "### Import delle librerie necessarie "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jMjL1B6iPcDM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys\n",
        "sys.path.append('/content/drive/My Drive/Python Files')\n",
        "\n",
        "import model as m\n",
        "import Decoding_Strategies\n",
        "import ModelUtility\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torchtext\n",
        "from torchtext.data import Field, BucketIterator\n",
        "from torchtext.datasets import TranslationDataset, Multi30k\n",
        "import torch.nn.functional as F\n",
        "  \n",
        "import spacy\n",
        "import numpy as np\n",
        "\n",
        "import random\n",
        "import math\n",
        "import time\n",
        "import queue as Q\n",
        "import operator\n",
        "\n",
        "import os\n",
        "import pandas as pd\n",
        "import re\n",
        "import json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ns21IBcP3VBB",
        "colab_type": "text"
      },
      "source": [
        "### Download Word Vecs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_oliqIvNPLyt",
        "colab_type": "code",
        "outputId": "d0c18ea8-dd9d-4f25-867b-c3834b0ac680",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "!unzip /content/drive/My\\ Drive/wiki-news-300d-1M.vec.zip -d /content/sample_data"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/drive/My Drive/wiki-news-300d-1M.vec.zip\n",
            "  inflating: /content/sample_data/wiki-news-300d-1M.vec  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xBwzdtLGa4WY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import io\n",
        "def load_vectors(fname):\n",
        "    fin = io.open(fname, 'r', encoding='utf-8', newline='\\n', errors='ignore')\n",
        "    n, d = map(int, fin.readline().split())\n",
        "    data = {}\n",
        "    for line in fin:\n",
        "        tokens = line.rstrip().split(' ')\n",
        "        data[tokens[0]] = map(float, tokens[1:])\n",
        "    return data\n",
        "\n",
        "embedded_vec = load_vectors(\"/content/sample_data/wiki-news-300d-1M.vec\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Kq1XlzlC4g1",
        "colab_type": "text"
      },
      "source": [
        "### Inizializzazione librerie e metodo per la tokenizzazione"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L8L-pzqM7NBn",
        "colab_type": "code",
        "outputId": "e00d204b-daad-446c-9e87-2bd448fcf7f9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "!pip install -U spacy\n",
        "#scarichiamo il tokenizer, necessario per rendere frasi liste di tokens\n",
        "!python -m spacy download en_core_web_sm\n",
        "#Load del modello dei singoli linguaggi\n",
        "spacy_en = spacy.load(\"en_core_web_sm\")\n",
        "\n",
        "#Tokenizer delle frasi in input tramite spacy\n",
        "def tokenize_en(text):\n",
        "    \"\"\"\n",
        "    Genera token della frase, creando una lista di stringhe\n",
        "    \"\"\"\n",
        "    return [tok.text for tok in spacy_en.tokenizer(text)]"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: spacy in /usr/local/lib/python3.6/dist-packages (2.2.4)\n",
            "Requirement already satisfied, skipping upgrade: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.3)\n",
            "Requirement already satisfied, skipping upgrade: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (4.38.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.18.2)\n",
            "Requirement already satisfied, skipping upgrade: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (3.0.2)\n",
            "Requirement already satisfied, skipping upgrade: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.2)\n",
            "Requirement already satisfied, skipping upgrade: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.2)\n",
            "Requirement already satisfied, skipping upgrade: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.6.0)\n",
            "Requirement already satisfied, skipping upgrade: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.1.3)\n",
            "Requirement already satisfied, skipping upgrade: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.4.1)\n",
            "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy) (46.1.3)\n",
            "Requirement already satisfied, skipping upgrade: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (7.4.0)\n",
            "Requirement already satisfied, skipping upgrade: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.0)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2020.4.5.1)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.8)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy) (1.6.0)\n",
            "Requirement already satisfied, skipping upgrade: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy) (3.1.0)\n",
            "Requirement already satisfied: en_core_web_sm==2.2.5 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz#egg=en_core_web_sm==2.2.5 in /usr/local/lib/python3.6/dist-packages (2.2.5)\n",
            "Requirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.6/dist-packages (from en_core_web_sm==2.2.5) (2.2.4)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (0.4.1)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (46.1.3)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (7.4.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (2.21.0)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (0.6.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (2.0.3)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (3.0.2)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.1.3)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.18.2)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (4.38.0)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (2.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (2020.4.5.1)\n",
            "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (1.6.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.1.0)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_sm')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EIjL_cgLTyL5",
        "colab_type": "text"
      },
      "source": [
        "###Generazione campi torchtext e dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ol5erWH4Z7HC",
        "colab_type": "text"
      },
      "source": [
        "Generiamo il Field, il dataset e l'intersezione tra dataset e vocabolario di FastText"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7dRK2__bPaPF",
        "colab_type": "code",
        "outputId": "ace725c4-806f-42e2-9fa3-ec57c217b2c4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "#Il campo in cui indichiamo come devono essere processate le frasi\n",
        "SRC = torchtext.data.Field(tokenize=tokenize_en,\n",
        "            init_token = '<sos>', \n",
        "            eos_token = '<eos>',\n",
        "            unk_token = 'unk',\n",
        "            fix_length = 41,\n",
        "            lower = True)\n",
        "\n",
        "#Creazione e Split del Dataset,partendo dal file CSV che contiene le frasi, basta codificare solo il SRC nel file\n",
        "#Tanto SRC==TRG\n",
        "dataset = torchtext.datasets.TranslationDataset(\"/content/drive/My Drive/new_tokenized_merged_dataset.csv\", exts= (\"\",\"\"), fields = (SRC,SRC))\n",
        "\n",
        "SRC.build_vocab(dataset, min_freq=3)\n",
        "len(SRC.vocab)\n",
        "\n",
        "def intersection(lst1, lst2): \n",
        "    lst3 = [value for value in lst1 if value in lst2] \n",
        "    return lst3 \n",
        "\n",
        "vocab = SRC.vocab.itos\n",
        "check = list(embedded_vec.keys())\n",
        "\n",
        "vocab_intersect = intersection(vocab,check)\n",
        "print(len(vocab_intersect))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "17965\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3qy37rLIZqIn",
        "colab_type": "text"
      },
      "source": [
        "Filtriamo il dataset levando tutte le frasi che contengono parole che non sono in FastText"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KxC9LFFK7oql",
        "colab_type": "code",
        "outputId": "a54c35d5-36e4-49d5-d82f-8a6fc7f2ab22",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "#Threshold per vedere se la frase ha abbastanza embeddings\n",
        "final_emb = dict()\n",
        "thresh = 1.0\n",
        "remove_from_train = []\n",
        "for example in dataset.examples:\n",
        "  temp_example = vars(example)['src']\n",
        "  percent = 0\n",
        "  for value in temp_example:\n",
        "    if value in vocab_intersect:\n",
        "      percent += 1\n",
        "  percent = percent/len(temp_example)\n",
        "  if percent < thresh:\n",
        "    remove_from_train.append(example)\n",
        "  # else:\n",
        "  #   for value in temp_example:\n",
        "  #     final_emb[value] = embedded_vec[value]\n",
        "      \n",
        "print(len(remove_from_train))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "632\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ueuBP9dd3Fug",
        "colab_type": "text"
      },
      "source": [
        "Rimuoviamo le frasi che non contengono le cui parole non fanno tutte parte di FastText e rigeneriamo il vocabolario."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bdMDfhWOEgTC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "57767abf-48a9-49b8-e773-cc242328ef2b"
      },
      "source": [
        "for value in remove_from_train:\n",
        "  try:\n",
        "    dataset.examples.remove(value)\n",
        "  except:\n",
        "    print(value)\n",
        "\n",
        "SRC.build_vocab(dataset, min_freq=3)\n",
        "print(\"Lunghezza del dizionario:\")\n",
        "print(len(SRC.vocab))\n",
        "print(\"Lunghezza del dataset:\")\n",
        "print(len(dataset.examples))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Lunghezza del dizionario:\n",
            "17809\n",
            "Lunghezza del dataset:\n",
            "43970\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tvF8GT8xZZ2e",
        "colab_type": "text"
      },
      "source": [
        "Generiamo la matrice di embeddings da passare direttamente al modello come layer di Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N-_G9sILRiQ0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "special_tokens = ['unk','<pad>','<sos>','<eos>']\n",
        "\n",
        "for value in SRC.vocab.itos:\n",
        "  if value not in special_tokens:\n",
        "      final_emb[value] = embedded_vec[value]\n",
        "    \n",
        "final_emb['<eos>'] = embedded_vec['eos']\n",
        "final_emb['<sos>'] = embedded_vec['sos']\n",
        "final_emb['unk'] = embedded_vec['unk']\n",
        "\n",
        "#Impostiamo il l'embedding di pad\n",
        "pad = [0]*300\n",
        "pad[0] = 1\n",
        "final_emb['<pad>'] = pad\n",
        "\n",
        "#Spacchettiamo le maps\n",
        "for key in final_emb.keys():\n",
        "  final_emb[key] = list(final_emb[key])\n",
        "\n",
        "#Prendo i pesi ordinati rispetto al dizionario, in questo\n",
        "#Modo l'ordine si conserva, necessario al corretto\n",
        "#Funzionamento degli embeddings.\n",
        "emb_list = []\n",
        "for value in SRC.vocab.itos:\n",
        "  emb_list.append(final_emb[value])\n",
        "\n",
        "#Porto i pesi in numpy_array\n",
        "emb_weigths = np.array(emb_list, dtype = float)\n",
        "\n",
        "#Salvo <sos> per usarlo come input dell'attn\n",
        "sos_emb = final_emb['<sos>']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R4uE1B4rY8mQ",
        "colab_type": "text"
      },
      "source": [
        "Generiamo Train, Test e Validation Data e successivamente liberiamo la RAM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BaO1IDzXu8Tv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Split del train, validation e test\n",
        "train_data, valid_data, test_data = dataset.split(split_ratio=[0.7,0.15,0.15])\n",
        "\n",
        "BATCH_SIZE = 64\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "train_iterator, valid_iterator, test_iterator = torchtext.data.BucketIterator.splits(\n",
        "    (train_data, valid_data, test_data), \n",
        "    batch_size = BATCH_SIZE, \n",
        "    device = device)\n",
        "\n",
        "#Cancello queste variabili liberando circa 19GB di RAM\n",
        "del(final_emb)\n",
        "del(check)\n",
        "del(embedded_vec)\n",
        "del(vocab_intersect)#Parametri della rete"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wFN6DA3x5Vpg",
        "colab_type": "text"
      },
      "source": [
        "###Creazione matrice One Hot per la BoW"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-wsdQ2bcud1Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_fwf(\"/content/drive/My Drive/wordlists/bow_economy.txt\")\n",
        "bow = []\n",
        "for value in df['business']:\n",
        "  bow.append(value)\n",
        "\n",
        "bow.append('business')\n",
        "\n",
        "indexes = []\n",
        "for value in bow:\n",
        "  indexes.append(SRC.vocab.stoi[value])\n",
        "\n",
        "one_hot = torch.zeros(len(indexes),len(SRC.vocab.itos)).cuda()\n",
        "\n",
        "word_index = 0\n",
        "for value in indexes:\n",
        "  one_hot[word_index,value] = 1  \n",
        "  word_index +=1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wn67SawXhnzf",
        "colab_type": "text"
      },
      "source": [
        "###Modello e inizializzazione del modello"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c4aTbHaFJX46",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Encoder, con definizione dell'architettura e del forward\n",
        "class Encoder(nn.Module):\n",
        "    def __init__(self,input_dim, dim, n_layers, dropout,emb_weights, device):\n",
        "        \n",
        "        super().__init__()\n",
        "        self.device = device\n",
        "        self.dim = dim\n",
        "        self.n_layers = n_layers\n",
        "        self.input_dim = input_dim\n",
        "        self.emb_weights = emb_weights\n",
        "\n",
        "        self.embedding = nn.Embedding(input_dim, dim)\n",
        "        self.embedding.weight = nn.Parameter(torch.from_numpy(emb_weights).to(torch.float))\n",
        "        self.embedding.weight.requires_grad= False\n",
        "\n",
        "        self.rnn = nn.LSTM(dim, dim, n_layers, dropout = dropout, batch_first=True)\n",
        "        \n",
        "        \n",
        "    def forward(self, src):      \n",
        " \n",
        "        #src = [time_step,batch_size]\n",
        "        src = torch.t(src)\n",
        "        embedded = self.embedding(src)\n",
        "        # print(\"Shape embedding size encoder:\")\n",
        "        # print(embedded.shape) # shape = (BS,TS,HDIM)\n",
        "        output, (hidden, cell) = self.rnn(embedded)\n",
        "\n",
        "        return hidden, cell, output\n",
        "\n",
        "\n",
        "\n",
        "class Attention(nn.Module):\n",
        "  def __init__(self, dim, att_dim,device,heads = 3):\n",
        "    \n",
        "    super().__init__()\n",
        "\n",
        "    self.dim = dim #Dimensione sia degli embedding che degli hidden_states\n",
        "    self.att_dim = att_dim #Lunghezza massima della stringa, necessaria alla creazione dello strato attn\n",
        "    self.device = device #GPU\n",
        "    self.heads = heads #N di attention\n",
        "    \n",
        "    #Prima matrice dei pesi dell'attention\n",
        "    self.w1 = torch.zeros(2 * self.dim , \n",
        "                          self.att_dim, \n",
        "                          device = device, \n",
        "                          requires_grad = True)\n",
        "    self.w1 = torch.nn.Parameter(self.w1)\n",
        "    \n",
        "    #Seconda matrice dell'attention\n",
        "    self.w2 = torch.zeros(self.att_dim, \n",
        "                          self.heads,\n",
        "                          device = device,\n",
        "                          requires_grad = True) \n",
        "    self.w2 = torch.nn.Parameter(self.w2)\n",
        " \n",
        "  def forward(self, encoder_outputs, prev_dec_output):\n",
        "    \n",
        "    #print(prev_dec_output.shape) # shape = (BS, 1(TS dei layer della rete), 300)\n",
        "    \n",
        "    #print(encoder_outputs.shape) # shape = (BS,TS,HDIM)\n",
        "\n",
        "    #Aumentiamo le dimensioni così da permettere il concat\n",
        "    prev_dec_output_expand = prev_dec_output.expand_as(encoder_outputs) # shape = (BS,TS,HDIM)\n",
        "\n",
        "    #Concateniamo l'econder_outputs con il precedente stato del decoder\n",
        "    encoder_outputs_concat = torch.cat((encoder_outputs, prev_dec_output_expand), 2) # shape = (BS, TS, 2*HDIM)\n",
        "\n",
        "    #Primo MatMul\n",
        "    att_op_1 = torch.matmul(encoder_outputs_concat, self.w1) # shape = (BS, TS, ATT_DIM)\n",
        "    att_op_1 = torch.tanh(att_op_1)\n",
        "\n",
        "    #Secondo MatMul\n",
        "    att_op_2 = torch.matmul(att_op_1, self.w2)  # shape = (BS, TS, HEADS)\n",
        "    att_op_2 = torch.tanh(att_op_2)\n",
        "    #Softmax sulla seconda dimesione della matrice di attention\n",
        "    att_weights = F.softmax(att_op_2,dim=1) #shape = (BS, TS, HEADS)\n",
        "    \n",
        "    #print(att_weights)\n",
        "    #Trasposta della matrice degli output dell'encoder\n",
        "    encoder_outputs_concat = encoder_outputs_concat.permute(0,2,1) # shape = (BS,2*HDIM,TS)\n",
        "    \n",
        "    #Terzo MatMul, tra encoder_outputs concatenato con il prev_dec_state e gli attn_weights\n",
        "    att_output = torch.matmul(encoder_outputs_concat, att_weights) # shape = (BS,2*HDIM,HEADS)\n",
        "    att_output = torch.tanh(att_output)\n",
        "    att_output = att_output.reshape(att_output.shape[0],att_output.shape[1]*att_output.shape[2]) # shape =(BS,2*HDIM*HEADS)\n",
        "    \n",
        "    return att_output\n",
        "\n",
        "\n",
        "class Decoder(nn.Module):\n",
        "  def __init__(self, dim, output_dim, dropout, n_layers,device,emb_weights,heads = 3):\n",
        "    super().__init__()\n",
        "\n",
        "    self.device = device #GPU\n",
        "    self.dim = dim #Dimensione sia degli embedding che degli hidden_states\n",
        "    self.output_dim = output_dim #Dimensione del vocabolario di arriva\n",
        "    self.n_layers = n_layers #numero di layer dell'encoder\n",
        "    self.emb_weights = emb_weights #Pesi degli embeddings \n",
        "\n",
        "    self.w3 = torch.zeros(self.dim, self.output_dim, \n",
        "                          device = device, \n",
        "                          requires_grad = True) #Terza matrice dell'attention\n",
        "    self.w3 = torch.nn.Parameter(self.w3)\n",
        "\n",
        "    self.w4 = torch.zeros(self.dim * 6 + self.dim,\n",
        "                          self.dim, \n",
        "                          device = device, \n",
        "                          requires_grad = True) #Terza matrice dell'attention\n",
        "    self.w4 = torch.nn.Parameter(self.w4)\n",
        "\n",
        "    self.rnn = nn.LSTM(self.dim,\n",
        "                       self.dim, \n",
        "                       n_layers, dropout = dropout, \n",
        "                       batch_first = True) #LSTM\n",
        "\n",
        "    self.embedding = nn.Embedding(output_dim, dim)\n",
        "    self.embedding.weight = nn.Parameter(torch.from_numpy(emb_weights).to(torch.float))\n",
        "    self.embedding.weight.requires_grad= False\n",
        "\n",
        "  def forward(self,att_output, input,hidden, cell):\n",
        "    \n",
        "    #Da implementare embeddings\n",
        "    emb = self.embedding(input)\n",
        "    #Nel caso della Beam Search abbiamo meno dimensioni\n",
        "    try:\n",
        "      if (len(emb.shape) == 1):\n",
        "        decoder_input = torch.cat((att_output, emb.unsqueeze(0)),1).unsqueeze(0) # shape = (1,BS,2*HDIM*HEADS + HDIM)\n",
        "      else:\n",
        "        decoder_input = torch.cat((att_output, emb),1).unsqueeze(0) # shape = (1,BS,2*HDIM*HEADS + HDIM)\n",
        "    except:\n",
        "      print(att_output.shape)\n",
        "      print(emb.shape)\n",
        "    decoder_input = decoder_input.permute(1,0,2) # shape = (BS, 1, 2*HDIM*HEADS + HDIM)\n",
        "    \n",
        "    decoder_input_w4 = torch.matmul(decoder_input,self.w4) # shape = (BS, 1, HDIM)\n",
        "    #decoder_input_w4 = torch.tanh(decoder_input_w4)\n",
        "\n",
        "    dec_state, (hidden, cell) = self.rnn(decoder_input_w4,(hidden,cell))\n",
        "\n",
        "    output = torch.matmul(dec_state,self.w3)\n",
        "\n",
        "    return output.squeeze(1), hidden, cell, dec_state\n",
        "\n",
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self, encoder, decoder, attention, device, sos_emb,max_length, beam_width = 3):\n",
        "        super().__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.attention = attention\n",
        "        self.device = device\n",
        "        self.sos_emb = sos_emb\n",
        "        self.max_length = max_length\n",
        "        self.beam_width = beam_width\n",
        "        assert encoder.dim == decoder.dim, \\\n",
        "            \"Hidden dimensions of encoder and decoder must be equal!\"\n",
        "        assert encoder.n_layers == decoder.n_layers, \\\n",
        "            \"Encoder and decoder must have equal number of layers!\"\n",
        "\n",
        "#Dimensione del dizionario dell'input\n",
        "#Dimensione del dizionario di output\n",
        "#Dimensione interna, sia degli embeddings che degli hidden states\n",
        "#Numero di layers delle LSTM dell'Encoder e del Decoder\n",
        "#Probabiltà di Dropout delle LSTM\n",
        "#Grandezza massima delle stringhe in entrata(N di token massimi)\n",
        "#Matrice degli Embeddings\n",
        "#Device su cui far runnare tutto, si spera GPU\n",
        "def model_initialization(INPUT_DIM,OUTPUT_DIM,DIMENSION,N_LAYERS, DROPOUT, MAX_SIZE,emb_weights, device,sos_emb):\n",
        "    #Dichiarazione del modello\n",
        "    enc = Encoder(INPUT_DIM, DIMENSION, N_LAYERS, DROPOUT,emb_weights,device)\n",
        "    attn = Attention(DIMENSION, MAX_SIZE, device)\n",
        "    dec = Decoder(DIMENSION,OUTPUT_DIM,DROPOUT,N_LAYERS,device,emb_weights)\n",
        "    model = Seq2Seq(enc, dec, attn, device, sos_emb, MAX_SIZE).to(device)\n",
        "   \n",
        "    model.cuda()\n",
        "\n",
        "    for name, param in model.named_parameters():\n",
        "        nn.init.uniform_(param.data, -0.09, 0.09)\n",
        "\n",
        "    return model\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BH9miZLPazG9",
        "colab_type": "text"
      },
      "source": [
        "Per adesso non utilizziamo criteri di ottimizzazione (Nel nostro caso CrossEntropyLoss)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eHwiqExoVoAP",
        "colab_type": "code",
        "outputId": "903a26f2-f2b3-45a4-8160-28a092ea0ff4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "#Input e output sono due OHE sui vocabolari di partenza e di arrivo\n",
        "INPUT_DIM = len(SRC.vocab)\n",
        "OUTPUT_DIM = INPUT_DIM\n",
        "#Dimensione dello strato di embedding, anche se io dovrei usare fastText\n",
        "\n",
        "#Dimensione degli strati nascosti, uguale per ENC|DEC, vale anche per gli embeddings\n",
        "DIMENSION = 300\n",
        "N_LAYERS = 1\n",
        "\n",
        "#Probabilità del dropout, applicato sullo strato di embedding\n",
        "DROPOUT = 0.5\n",
        "modello = model_initialization(INPUT_DIM,OUTPUT_DIM,DIMENSION, N_LAYERS,DROPOUT,41,emb_weigths,device,sos_emb)\n",
        "\n",
        "#Definiamo il criterio di Loss, controllando di indicare su quali elementi non applicarlo, in questo caso il padding\n",
        "SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token]\n",
        "\n",
        "\n",
        "#criterion = nn.CrossEntropyLoss(ignore_index = SRC_PAD_IDX)\n",
        "\n",
        "def count_parameters(modello):\n",
        "    return sum(p.numel() for p in modello.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(modello):,} trainable parameters')\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs\n",
        "\n",
        "#Metodo di ottimizzazione dei parametri della rete\n",
        "optimizer = optim.Adam(modello.parameters(), lr = 0.001)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The model has 7,442,223 trainable parameters\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:50: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
            "  \"num_layers={}\".format(dropout, num_layers))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36YMOnx0aqB0",
        "colab_type": "code",
        "outputId": "30fb5a47-b1d0-4816-c1c5-3a44429e6ea7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "modello.load_state_dict(torch.load(\"/content/drive/My Drive/new_model_weigths_13_04.pt\"))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u6C2g8YRZhSc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Blocchiamo tutti i parametri del modello, tanto è già addestrato\n",
        "for params in modello.parameters():\n",
        "  params.requires_grad = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F6WBzUNkhuyE",
        "colab_type": "text"
      },
      "source": [
        "###Metodi per la perturbazione della predizione"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EAlR8bNaa7wd",
        "colab_type": "text"
      },
      "source": [
        "Metodo che ritorna una variabile trainable utilizzando i dati dell'input"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eikqHz09hCkm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.autograd import Variable\n",
        "\n",
        "def to_var(x, requires_grad=False, volatile=False):\n",
        "    if torch.cuda.is_available():\n",
        "        x = x.cuda()\n",
        "    return Variable(x, requires_grad=requires_grad, volatile=volatile)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qIaZKKXZbEFc",
        "colab_type": "text"
      },
      "source": [
        "Primo metodo della perturbazione, cicla su tutto l'iteratore e stampa frase perturbata e frase target"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_mifUgNAUIM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "\n",
        "#Ritorna una lista di frasi perturbate, prese dall'iterator\n",
        "def generate_perturbed_prediction(model,iterator,SRC,one_hot):\n",
        "\n",
        "      # Per ogni batch nell'iteratore\n",
        "      for i, batch in enumerate(test_iterator):\n",
        "        src = batch.src\n",
        "        trg = batch.trg\n",
        "\n",
        "        #Per ogni frase nella batch\n",
        "        for j in range(len(src)):\n",
        "          \n",
        "          enc_input = src[:,j].unsqueeze(0)\n",
        "          output_perturb = perturb_phrase(model, enc_input,one_hot)\n",
        "          \n",
        "          #Stampiamo la frase originale\n",
        "          phrase = \"\"\n",
        "          print(\"Target phrase\")\n",
        "          for val in enc_input[0]:\n",
        "            if(val != 3 and val != 1):\n",
        "              phrase +=  \" \" + SRC.vocab.itos[val]\n",
        "          print(phrase)\n",
        "          \n",
        "          #Stampiamo la frase perturbata\n",
        "          phrase = \"\"\n",
        "          print(\"Phrase predicted:\")\n",
        "          for val in output_perturb:\n",
        "            if(val != 3 and val != 1):\n",
        "              phrase +=  \" \" + SRC.vocab.itos[val]\n",
        "          print(phrase)\n",
        "          print(\" \")\n",
        "          \n",
        "          #Giusto per vedere cosa stampa sennò stampa troppo velocemente\n",
        "          time.sleep(1)\n",
        "\n",
        "      return;"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0GF-5aAHbLZk",
        "colab_type": "text"
      },
      "source": [
        "Metodo che ritorna la frase perturbata, prendendo in input una frase, il modello e la BoW"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RN-sxapgBHI2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Ritorna la frase perturbata\n",
        "def perturb_phrase(model,enc_input,one_hot):\n",
        "  #Lista delle predizioni perturbate\n",
        "  output = []\n",
        "\n",
        "  #Frase in input\n",
        "  enc_input = torch.t(enc_input)\n",
        "\n",
        "  #hidden shape = torch.Size([1, 41, 300])\n",
        "  #cell shape = torch.Size([1, 41, 300])\n",
        "  #encoder_output shape = torch.Size([1, 41, 300])\n",
        "  hidden, cell, encoder_outputs = model.encoder(enc_input)\n",
        "  \n",
        "  #Piccola constante necessaria\n",
        "  small_const = 1e-15\n",
        "  sos_emb_matrix = torch.tensor(model.sos_emb,device=model.device)\n",
        "  sos_emb_matrix = sos_emb_matrix.expand_as(torch.zeros(1,1, 300))\n",
        "\n",
        "  #Generiamo gli input del decoder al passo 0\n",
        "  dec_state_t = sos_emb_matrix\n",
        "  dec_input = enc_input[0,0]\n",
        "  output.append(dec_input.item())\n",
        "\n",
        "  #Layer di Attention\n",
        "  att_output_original = model.attention(encoder_outputs, dec_state_t)\n",
        "  output_original, hidden_original, cell_original, dec_state_original = model.decoder(att_output_original,\n",
        "                                                                                          dec_input, \n",
        "                                                                                          hidden, cell)\n",
        "  #Riassegniamo tutte le componenti per il prossimo timestep \n",
        "  hidden = hidden_original\n",
        "  cell = cell_original\n",
        "  dec_state_t = dec_state_original \n",
        "  dec_input = output_original.argmax(1)\n",
        "  output.append(dec_input[0].item())\n",
        "\n",
        "  for i in range(model.max_length - 1):\n",
        "      #Questi sono gli originali, ad ogni passo li andremo a modificare \n",
        "      #all'interno di questo ciclo\n",
        "\n",
        "      #Layer di Attention\n",
        "      att_output_original = model.attention(encoder_outputs, dec_state_t)\n",
        "\n",
        "      #Layer di Decoding\n",
        "      #hidden shape = [1,1,300]\n",
        "      #cell shape = [1,1,300]\n",
        "      output_original, hidden_original, cell_original, dec_state_original = model.decoder(att_output_original,\n",
        "                                                                                          dec_input, \n",
        "                                                                                          hidden, cell)\n",
        "      #primi stati perturbati\n",
        "      perturbed_hidden = hidden.detach().clone()\n",
        "      perturbed_dec_state = dec_state_t.detach().clone()\n",
        "      perturbed_encoder_outputs = encoder_outputs.detach().clone()\n",
        "\n",
        "      #Genero variabili trainable\n",
        "      perturbed_hidden = to_var(perturbed_hidden,True)\n",
        "      perturbed_dec_state = to_var(perturbed_dec_state,True)\n",
        "      perturbed_encoder_outputs = to_var(perturbed_encoder_outputs, True)\n",
        "\n",
        "      #Calcoliamo gli hidden state e i dec state perturbati\n",
        "      perturbed_hidden, perturbed_dec_state, perturbed_encoder_outputs = perturb_time_step(perturbed_hidden,\n",
        "                                                                perturbed_dec_state, \n",
        "                                                                model,\n",
        "                                                                one_hot,\n",
        "                                                                perturbed_encoder_outputs,\n",
        "                                                                dec_input,\n",
        "                                                                cell,\n",
        "                                                                output_original,\n",
        "                                                                small_const\n",
        "                                                                )\n",
        "      \n",
        "      #Calcoliamo l'output finale perturbato \n",
        "      perturbed_att_output = model.attention(perturbed_encoder_outputs, perturbed_dec_state)\n",
        "\n",
        "      perturbed_output,_,_,_ = model.decoder(perturbed_att_output,\n",
        "                                             dec_input,\n",
        "                                             hidden,cell)\n",
        "      \n",
        "      #Calcoliamo le probabilità dei due modelli, per fare successivamente il blend\n",
        "      original_probs = F.softmax(output_original, dim = 1)\n",
        "      perturbed_probs = F.softmax(perturbed_output, dim = 1)\n",
        "      top1_perturb = perturbed_probs.argmax(1)\n",
        "\n",
        "      #Blend delle probabilità tramite post norm geometric mean fusion\n",
        "      gm_scale = 0.9\n",
        "      final_probs = (perturbed_probs ** gm_scale) * (original_probs ** (1 - gm_scale))\n",
        "\n",
        "      #Rinormalizziamolo nel caso in cui non lo sia dopo il blend\n",
        "      if(torch.sum(final_probs) <= 1):\n",
        "        final_probs = final_probs / torch.sum(final_probs)\n",
        "\n",
        "      #Riassegniamo tutte le componenti per il prossimo timestep \n",
        "      hidden = hidden_original\n",
        "      cell = cell_original\n",
        "      dec_state_t = dec_state_original \n",
        "      dec_input = final_probs.argmax(1)\n",
        "      # if(top1_perturb.item() != 3):\n",
        "      #   print(top1_perturb.item())\n",
        "      #   print(dec_input[0].item())\n",
        "      output.append(dec_input[0].item())\n",
        "\n",
        "  return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yXzKRWJTbS6I",
        "colab_type": "text"
      },
      "source": [
        "Metodo che perturba un singolo TimeStep e ritorna l'hidden_state, il decoder_state e l'encoder_output perturbati per un certo numero di TimeStep"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tCNF_Bu0w-v4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def perturb_time_step(perturbed_hidden, \n",
        "                      perturbed_dec_state, \n",
        "                      model,\n",
        "                      one_hot,\n",
        "                      perturbed_encoder_outputs,\n",
        "                      dec_input,\n",
        "                      cell,\n",
        "                      output_original,\n",
        "                      small_const):\n",
        "  \n",
        "  #Genero variabili trainable\n",
        "  perturbed_hidden = to_var(perturbed_hidden,True)\n",
        "  perturbed_dec_state = to_var(perturbed_dec_state,True)\n",
        "  \n",
        "  #Inizializziamo il le matrici che conterranno i gradienti\n",
        "  #Ad ogni passo sommiamo allo stesso hidden un gradiente accumulato \n",
        "  hidden_grad_sum = torch.zeros(1,1, device= model.device).expand_as(perturbed_hidden).clone()\n",
        "  dec_state_grad_sum = torch.zeros(1,1, device = model.device).expand_as(perturbed_dec_state).clone()\n",
        "  enc_output_grad_sum = torch.zeros(1,1, device = model.device).expand_as(perturbed_encoder_outputs).clone()\n",
        "\n",
        "  # print(\"****************************\")\n",
        "\n",
        "  #Per ogni passo dell'interazione\n",
        "  for i in range(10):\n",
        "    #Sommiamo le perturbazioni agli hidden\n",
        "    perturbed_hidden.data += hidden_grad_sum\n",
        "    perturbed_dec_state.data += dec_state_grad_sum\n",
        "    perturbed_encoder_outputs.data += enc_output_grad_sum\n",
        "    \n",
        "    loss = 0.0\n",
        "\n",
        "    #Generiamo gli output del modello perturbato\n",
        "    att_output_perturbed = model.attention(perturbed_encoder_outputs, perturbed_dec_state)\n",
        "    output_perturbed,_,_,_ = model.decoder(att_output_perturbed, dec_input, perturbed_hidden, cell)\n",
        "    \n",
        "    #Softmax per farlo diventare una vera distribuzione\n",
        "    probabs_perturbed = F.softmax(output_perturbed,dim = 1)\n",
        "    #Generazione BOW Loss\n",
        "    #Cancella le parole nella bow certe volte\n",
        "    #Altre volte non si sa perchè la somma fa un numero che è quasi 1, ma non è possibile\n",
        "    good_logits = torch.mm(probabs_perturbed,torch.t(one_hot))\n",
        "    good_logits = good_logits.clone()\n",
        "    loss_word = torch.sum(good_logits.squeeze(0)).clone()\n",
        "    #Per adesso mettiamo un \"cerotto\"\n",
        "    if(loss_word.item() >= 0.9):\n",
        "      loss_word = 0.0\n",
        "      # print(\"Ricalcolo la somma\")\n",
        "      #Da capire perchè la BOW si bugga, dato che ci sono certe volte degli elementi troppo grandi\n",
        "      for value in good_logits.squeeze(0):\n",
        "        \n",
        "        if value < 0.9:\n",
        "          loss_word += value\n",
        "      # print(\"Somma ricalcolata\")\n",
        "      # print(loss_word)\n",
        "    #print(loss_word)\n",
        "    # if(probabs_perturbed.argmax(1).item() != 3):\n",
        "    #   print(\" \")\n",
        "    #   print(\"Somma delle probabilità della bow\")\n",
        "    #   print(loss_word)\n",
        "    loss_word = torch.log(loss_word)\n",
        "    #Sommiamo la bow_loss alla loss totale\n",
        "    loss += loss_word\n",
        "\n",
        "    #Calcolo della KL \n",
        "    kl_scale = 0.01\n",
        "    p_original = F.softmax(output_original, dim = 1)\n",
        "    p_original = p_original + small_const * (p_original <= small_const).type(torch.FloatTensor).cuda().detach()\n",
        "    correction = small_const * (probabs_perturbed <= small_const).type(torch.FloatTensor).cuda().detach()\n",
        "    corrected_probabs = probabs_perturbed + correction.detach()\n",
        "    kl_loss = kl_scale * ((corrected_probabs * (corrected_probabs / p_original).log()).sum())\n",
        "    # if(probabs_perturbed.argmax(1).item() != 3):\n",
        "    #   print(\"Parola predetta:\")\n",
        "    #   print(probabs_perturbed.argmax(1).item())\n",
        "    #   print(\"BOW Loss\")\n",
        "    #   print(loss_word)\n",
        "    loss -= kl_loss\n",
        "\n",
        "    #Passo di backward\n",
        "    loss.backward()\n",
        "    \n",
        "    #Gradiente normalizzato\n",
        "    g_n_hidden_state = torch.norm(perturbed_hidden.grad) + small_const\n",
        "    g_n_dec_state = torch.norm(perturbed_dec_state.grad) + small_const\n",
        "    g_n_enc_output = torch.norm(perturbed_encoder_outputs) + small_const\n",
        "    \n",
        "    #Inizializzazione dei parametri della rete\n",
        "    stepsize = 0.02\n",
        "    gamma = 1.5\n",
        "\n",
        "    #Aggiorniamo il delta t\n",
        "    hidden_grad_sum += stepsize * (perturbed_hidden.grad / g_n_hidden_state ** gamma) \n",
        "    dec_state_grad_sum += stepsize * (perturbed_dec_state.grad / g_n_dec_state ** gamma)\n",
        "    enc_output_grad_sum += stepsize * (perturbed_encoder_outputs.grad / g_n_enc_output ** gamma)\n",
        "\n",
        "    #Azzeriamo il gradiente di ogni variabile\n",
        "    perturbed_hidden.grad.data.zero_()\n",
        "    perturbed_dec_state.grad.data.zero_()\n",
        "    perturbed_encoder_outputs.grad.data.zero_()\n",
        "\n",
        "    #Nel dubbio, detach delle matrici \n",
        "    perturbed_hidden = perturbed_hidden.detach()\n",
        "    perturbed_dec_state = perturbed_dec_state.detach()\n",
        "    perturbed_encoder_outputs = perturbed_encoder_outputs.detach()\n",
        "\n",
        "    #Genero variabili trainable\n",
        "    perturbed_hidden = to_var(perturbed_hidden,True)\n",
        "    perturbed_dec_state = to_var(perturbed_dec_state,True)\n",
        "    perturbed_encoder_outputs = to_var(perturbed_encoder_outputs,True)\n",
        "\n",
        "  #Torniamo i stati perturbati\n",
        "  perturbed_hidden.data += hidden_grad_sum\n",
        "  perturbed_dec_state.data += dec_state_grad_sum\n",
        "  perturbed_encoder_outputs.data += enc_output_grad_sum\n",
        "\n",
        "  return perturbed_hidden, perturbed_dec_state, perturbed_encoder_outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-V26DWTa97SD",
        "colab_type": "text"
      },
      "source": [
        "Trovato bug su sum, spesso somma prendendo direttamente i valori da tutta la distribuzione piuttosto che dalla BoW, infatti la somma viene 1 o 0.999, il che non è possibile se prendo una partizione di un gruppo normalizzato.\n",
        "\n",
        "Questo si presenta spesso se la parola in a quel TS nel SRC è una parola della BoW.\n",
        "\n",
        "Per adesso ci ho messo un cerotto: I valori troppo grandi vengono rimossi dalla somma, da capire come risolvere al meglio.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "nljFqoWqJarq",
        "outputId": "23875b56-7044-4d3b-bee0-1934276a609d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "generate_perturbed_prediction(modello,test_iterator,SRC,one_hot)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Target phrase\n",
            " <sos> hard cases make bad law ...\n",
            "Phrase predicted:\n",
            " <sos> hard cases make bad law ... just\n",
            " \n",
            "Target phrase\n",
            " <sos> power of money and economy !\n",
            "Phrase predicted:\n",
            " <sos> power of money and economy !\n",
            " \n",
            "Target phrase\n",
            " <sos> 's cruise cock - up via\n",
            "Phrase predicted:\n",
            " <sos> 's working isolation - up via\n",
            " \n",
            "Target phrase\n",
            " <sos> craziest alert i ever got !\n",
            "Phrase predicted:\n",
            " <sos> craziest alert i ever got !\n",
            " \n",
            "Target phrase\n",
            " <sos> 6.6 million americans file jobless claims\n",
            "Phrase predicted:\n",
            " <sos> 6.6 million americans file jobless claims\n",
            " \n",
            "Target phrase\n",
            " <sos> global bailouts to fight the -\n",
            "Phrase predicted:\n",
            " <sos> global manufacturers to fight the -\n",
            " \n",
            "Target phrase\n",
            " <sos> i just be minding my business\n",
            "Phrase predicted:\n",
            " <sos> i just be minding my business\n",
            " \n",
            "Target phrase\n",
            " <sos> it 's a lonely beach !\n",
            "Phrase predicted:\n",
            " <sos> it 's a lonely beach !\n",
            " \n",
            "Target phrase\n",
            " <sos> mines close as tightens controls -19\n",
            "Phrase predicted:\n",
            " <sos> tools close as stories free concerns just\n",
            " \n",
            "Target phrase\n",
            " <sos> maybe he 's hanging out with\n",
            "Phrase predicted:\n",
            " <sos> maybe he 's unemployment out with\n",
            " \n",
            "Target phrase\n",
            " <sos> bayern resume train amidst coronavirus pandemic\n",
            "Phrase predicted:\n",
            " <sos> bayern resume train amidst coronavirus pandemic\n",
            " \n",
            "Target phrase\n",
            " <sos> in times ? for who ?\n",
            "Phrase predicted:\n",
            " <sos> in times ? for who ?\n",
            " \n",
            "Target phrase\n",
            " <sos> 7 affordable advertising strategies for your\n",
            "Phrase predicted:\n",
            " <sos> 7 out advertising strategies for your\n",
            " \n",
            "Target phrase\n",
            " <sos> distracted by and golf you fool\n",
            "Phrase predicted:\n",
            " <sos> distracted by and free you fool\n",
            " \n",
            "Target phrase\n",
            " <sos> are the news before it happens\n",
            "Phrase predicted:\n",
            " <sos> are the news before it happens field\n",
            " \n",
            "Target phrase\n",
            " <sos> how does a ventilator work ?\n",
            "Phrase predicted:\n",
            " <sos> how does a ventilator work ?\n",
            " \n",
            "Target phrase\n",
            " <sos> how accurate is this info ?\n",
            "Phrase predicted:\n",
            " <sos> how investors is this info ?\n",
            " \n",
            "Target phrase\n",
            " <sos> i agree 110 % ! !\n",
            "Phrase predicted:\n",
            " <sos> i agree lifts % ! !\n",
            " \n",
            "Target phrase\n",
            " <sos> american in a fun song .\n",
            "Phrase predicted:\n",
            " <sos> american in a fun song . aimed\n",
            " \n",
            "Target phrase\n",
            " <sos> minding my own business 😉 😉\n",
            "Phrase predicted:\n",
            " <sos> minding my own business unemployment unemployment\n",
            " \n",
            "Target phrase\n",
            " <sos> not on business there is nt\n",
            "Phrase predicted:\n",
            " <sos> not on business there is nt just\n",
            " \n",
            "Target phrase\n",
            " <sos> noted doctor dies of coronavirus in\n",
            "Phrase predicted:\n",
            " <sos> noted doctor dies of coronavirus in 472 positive\n",
            " \n",
            "Target phrase\n",
            " <sos> my latest update photo ig :\n",
            "Phrase predicted:\n",
            " <sos> my latest update photo growth :\n",
            " \n",
            "Target phrase\n",
            " <sos> read read read read read read\n",
            "Phrase predicted:\n",
            " <sos> read read read read read read\n",
            " \n",
            "Target phrase\n",
            " <sos> social reproduction and the pandemic .\n",
            "Phrase predicted:\n",
            " <sos> social reproduction and the pandemic . |\n",
            " \n",
            "Target phrase\n",
            " <sos> cost control tools in business management\n",
            "Phrase predicted:\n",
            " <sos> cost control tools in business management 176\n",
            " \n",
            "Target phrase\n",
            " <sos> your * is clearly insane .\n",
            "Phrase predicted:\n",
            " <sos> your * is clearly insane .\n",
            " \n",
            "Target phrase\n",
            " <sos> the president and the plague via\n",
            "Phrase predicted:\n",
            " <sos> the president and the plague via 176\n",
            " \n",
            "Target phrase\n",
            " <sos> we owe them a lot ...\n",
            "Phrase predicted:\n",
            " <sos> we owe them a lot ...\n",
            " \n",
            "Target phrase\n",
            " <sos> stop the transmission of corona virus\n",
            "Phrase predicted:\n",
            " <sos> stop the transmission of corona virus tho\n",
            " \n",
            "Target phrase\n",
            " <sos> tell the fucking truth . by\n",
            "Phrase predicted:\n",
            " <sos> tell the fucking truth . claim\n",
            " \n",
            "Target phrase\n",
            " <sos> insurance premiums could spike due to\n",
            "Phrase predicted:\n",
            " <sos> insurance premiums could spike due to\n",
            " \n",
            "Target phrase\n",
            " <sos> never waste a good recession .\n",
            "Phrase predicted:\n",
            " <sos> never jobs a good recession . positive\n",
            " \n",
            "Target phrase\n",
            " <sos> are switching to due to -\n",
            "Phrase predicted:\n",
            " <sos> are switching to due to -\n",
            " \n",
            "Target phrase\n",
            " <sos> ammon bundy grifters strike again .\n",
            "Phrase predicted:\n",
            " <sos> ammon bundy grifters strike again . regan\n",
            " \n",
            "Target phrase\n",
            " <sos> this is 👍 ➡ ️ |\n",
            "Phrase predicted:\n",
            " <sos> this is 👍 ➡ ️ |\n",
            " \n",
            "Target phrase\n",
            " <sos> italy retail sales rise in february\n",
            "Phrase predicted:\n",
            " <sos> italy retail sales rise in february\n",
            " \n",
            "Target phrase\n",
            " <sos> sells its registry business to godaddy\n",
            "Phrase predicted:\n",
            " <sos> sells its registry business to godaddy rt\n",
            " \n",
            "Target phrase\n",
            " <sos> stay smiling with this morning 😊\n",
            "Phrase predicted:\n",
            " <sos> stay smiling with this morning 😊\n",
            " \n",
            "Target phrase\n",
            " <sos> trump : coronavirus task force update\n",
            "Phrase predicted:\n",
            " <sos> trump : coronavirus task force update\n",
            " \n",
            "Target phrase\n",
            " <sos> the president and the plague via\n",
            "Phrase predicted:\n",
            " <sos> the president and the plague via 176\n",
            " \n",
            "Target phrase\n",
            " <sos> . warns against easing measures too early\n",
            "Phrase predicted:\n",
            " <sos> . warns against easing measures too early\n",
            " \n",
            "Target phrase\n",
            " <sos> lost book of herbal remedies click here\n",
            "Phrase predicted:\n",
            " <sos> lost book of herbal remedies hidden here just\n",
            " \n",
            "Target phrase\n",
            " <sos> how to move quickly into a !\n",
            "Phrase predicted:\n",
            " <sos> how to move quickly into a !\n",
            " \n",
            "Target phrase\n",
            " <sos> remaining close while avoiding physical proximity .\n",
            "Phrase predicted:\n",
            " <sos> remaining close while avoiding legal proximity . |\n",
            " \n",
            "Target phrase\n",
            " <sos> so is starting to invade the world\n",
            "Phrase predicted:\n",
            " <sos> so is starting to protect the world\n",
            " \n",
            "Target phrase\n",
            " <sos> amsterdam launches next stage of its programme\n",
            "Phrase predicted:\n",
            " <sos> amsterdam launches congress stage of its unemployment\n",
            " \n",
            "Target phrase\n",
            " <sos> coronavirus crisis : snapshot of new york\n",
            "Phrase predicted:\n",
            " <sos> coronavirus crisis : snapshot of new york 472\n",
            " \n",
            "Target phrase\n",
            " <sos> that lies inside the question 😂 😂\n",
            "Phrase predicted:\n",
            " <sos> that lies some the question 😂 😂\n",
            " \n",
            "Target phrase\n",
            " <sos> biden spoke with trump about coronavirus response\n",
            "Phrase predicted:\n",
            " <sos> biden spoke with trump about coronavirus response\n",
            " \n",
            "Target phrase\n",
            " <sos> a fractured faces its greatest foe via\n",
            "Phrase predicted:\n",
            " <sos> a fractured unemployment its a foe via\n",
            " \n",
            "Target phrase\n",
            " <sos> spain industrial production decline slows in february\n",
            "Phrase predicted:\n",
            " <sos> spain supreme production decline slows in february\n",
            " \n",
            "Target phrase\n",
            " <sos> well done you know it makes sense\n",
            "Phrase predicted:\n",
            " <sos> well done you know it makes sense\n",
            " \n",
            "Target phrase\n",
            " <sos> when this is all over ... -19\n",
            "Phrase predicted:\n",
            " <sos> when this is all over ... -19 893\n",
            " \n",
            "Target phrase\n",
            " <sos> click to watch briefings of february 2020\n",
            "Phrase predicted:\n",
            " <sos> click to watch everywhere of bad 2020\n",
            " \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-5375d40a4eed>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mgenerate_perturbed_prediction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodello\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtest_iterator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mSRC\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mone_hot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-15-576ef5b6345f>\u001b[0m in \u001b[0;36mgenerate_perturbed_prediction\u001b[0;34m(model, iterator, SRC, one_hot)\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m           \u001b[0;31m#Giusto per vedere cosa stampa sennò stampa troppo velocemente\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m           \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m       \u001b[0;32mreturn\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HflR81-Qyw7M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "utils = ModelUtility.ModelUtility()\n",
        "utils.predict(modello,test_iterator, SRC)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}