datasets_elaborations.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Dataset 50-50.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "gS8PnAyYvv36"
      },
      "source": [
        "# import needed libraries \n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import random\n",
        "import json"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GC6P2XHAZO7Z",
        "outputId": "f1c081ca-c419-49a0-f931-d4ac292ce688"
      },
      "source": [
        "# mount google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8M48YFmB6fyL"
      },
      "source": [
        "def split_in_two(df, max_captions_per_image):\n",
        "  '''\n",
        "  Slit_in_two split a dataframe with columns 'caption' and \n",
        "  'image_id' which into 2 dataframes of the same dimension, which contains respectively a\n",
        "  correct couple (image_id, caption) and wrong couple (image_id, caption)\n",
        "\n",
        "  Label (third column in the final dataframes): 1:correct couple  0: wrong couple\n",
        "  max_captions_per_image: is the max possible number of captions for a single image\n",
        "\n",
        "  '''\n",
        "  meta=int(df['image_id'].nunique()/2)\n",
        "  count=0\n",
        "  id_immagini=df['image_id'].unique()\n",
        "  dataset_p=pd.DataFrame(columns=[\"caption\",\"image_id\"])\n",
        "  dataset_n=pd.DataFrame(columns=[\"caption\",\"image_id\"])\n",
        "\n",
        "  for img_id in id_immagini:\n",
        "\n",
        "    if (count<meta):\n",
        "      new_row=df[df['image_id']==img_id].iloc[0]\n",
        "      dataset_p=dataset_p.append(new_row,ignore_index=True)\n",
        "      count+=1\n",
        "\n",
        "    if (count>=meta) and count < (2*meta):\n",
        "      n=random.randint(0,(df['image_id'].nunique()-max_captions_per_image))\n",
        "      caption=df[df['image_id']!=img_id].iloc[n].caption\n",
        "      new_row={'caption':caption,'image_id':img_id}\n",
        "      dataset_n=dataset_n.append(new_row,ignore_index=True)\n",
        "      count+=1\n",
        "\n",
        "  dataset_n['Label']=[0]*len(dataset_n)\n",
        "  dataset_p['Label']=[1]*len(dataset_p)\n",
        "  dataset_p['image_id'].apply(lambda id: int(id))\n",
        "\n",
        "  return dataset_p, dataset_n"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WrzQrW265M9Z"
      },
      "source": [
        "#read json dataset\n",
        "train_j = json.load(open(\"/content/drive/captions_train2014.json\")) \n",
        "validation_j = json.load(open(\"/content/drive/captions_val2014.json\")) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7wGhR7Nx5ZyS"
      },
      "source": [
        "#create dataframes\n",
        "valid = pd.DataFrame(validation_j[\"annotations\"]) \n",
        "train_annotations= pd.DataFrame(train_j[\"annotations\"]) \n",
        "valid = valid[[\"caption\", \"image_id\"]]\n",
        "train = train_annotations[[\"caption\", \"image_id\"]] "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n_LOLAWBOEN0"
      },
      "source": [
        "#create dataframes: _n for negative labels, _p for positive ones\n",
        "validation_p, validation_n = split_in_two(valid,7)\n",
        "train_p, train_n  = split_in_two(train,7)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1SijhozKO8S_"
      },
      "source": [
        "#merge the positive dataset with the negative dataset\n",
        "validation_np= validation_n.merge(validation_p, how=\"outer\")\n",
        "train_np = train_n.merge(train_p, how=\"outer\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zFoA3AiBOC8G"
      },
      "source": [
        "validation_np.to_csv(\"/content/drive/dataval_n.csv\", index=False)\n",
        "train_np.to_csv(\"/content/drive/trainval_p.csv\", index=False)  "
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}