2_FB_Featurization_.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=red><p style=\"font-size:32px;text-align:center\"><b>FB Featurization</b></p></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pylab as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import pickle\n",
    "import networkx as nx\n",
    "import math\n",
    "import os\n",
    "\n",
    "from pandas import HDFStore,DataFrame\n",
    "from pandas import read_hdf\n",
    "from scipy.sparse.linalg import svds, eigs\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=blue><h3>Reading Data</h3></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_graph = nx.read_edgelist('D:/Applied_Ai/Case Studies/Facebook/My Work/after_eda/train_pos_after_eda.csv', delimiter=',', create_using=nx.DiGraph(), nodetype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: \n",
      "Type: DiGraph\n",
      "Number of nodes: 1780722\n",
      "Number of edges: 7550015\n",
      "Average in degree:   4.2399\n",
      "Average out degree:   4.2399\n"
     ]
    }
   ],
   "source": [
    "print(nx.info(train_graph))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=blue><h3>Similarity Measures</h3></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>1. Jaccard Distance</h4></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation}\n",
    "j = \\frac{|X\\cap Y|}{|X \\cup Y|} \n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#jaccard_distance_for_followees_\n",
    "def jaccard_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0 | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a))).intersection(len(set(train_graph.successors(b)))))/\\\n",
    "                        (len(set(train_graph.successors(a))).union(len(set(train_graph.successors(b)))))\n",
    "    except:\n",
    "        return 0\n",
    "    return sim        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(jaccard_for_followees(13567,12464))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#jaccard_distance_for_followers_\n",
    "def jaccard_for_followers(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.predecessors(a))) == 0 | len(set(train_graph.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a))).intersection(len(set(train_graph.predecessors(b)))))/\\\n",
    "                        (len(set(train_graph.predecessors(a))).union(len(set(train_graph.predecessors(b)))))\n",
    "    except:\n",
    "        return 0\n",
    "    return sim        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(jaccard_for_followers(13567,12464))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>2. Cosine Distance</h4></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation}\n",
    "CosineDistance = \\frac{|X\\cap Y|}{|X|\\cdot|Y|} \n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cosine_distance_for_followees_\n",
    "def cosine_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(cosine_for_followees(13567,12464))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cosine_distance_for_followers_\n",
    "def cosine_for_followers(a,b):\n",
    "    try:\n",
    "        \n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                     (math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(cosine_for_followers(2,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=blue><h3>Ranking Measures</h3></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>Page Ranking</h4></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/page_rank.p'):\n",
    "    pr = nx.pagerank(train_graph, alpha=0.85)\n",
    "    pickle.dump(pr, open('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/page_rank.p','wb'))\n",
    "else:\n",
    "    pr = pickle.load(open('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/page_rank.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Min 1.6556497245737814e-07\n",
      "Max 2.7098251341935827e-05\n",
      "Mean 5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "print('Min', pr[min(pr, key=pr.get)])\n",
    "print('Max', pr[max(pr, key=pr.get)])\n",
    "print('Mean', float(sum(pr.values()))/len(pr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "#for imputing to nodes which are not there in Train data\n",
    "mean_pr = float(sum(pr.values()))/len(pr)\n",
    "print(mean_pr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=blue><h3>Other Graph Features</h3></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>1. Shortest Path</h4></font>\n",
    "\n",
    "Getting shortest path between two nodes, if nodes have direct path i.e directly connected then we are removing that edge and calculating path."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#if has direct edge then deleting that edge and calculating shortest path\n",
    "def compute_shortest_path_length(a,b):\n",
    "    p=-1\n",
    "    try:\n",
    "        if train_graph.has_edge(a,b):\n",
    "            train_graph.remove_edge(a,b)\n",
    "            p= nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "            train_graph.add_edge(a,b)\n",
    "        else:\n",
    "            p= nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "        return p\n",
    "    except:\n",
    "        return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(compute_shortest_path_length(27458,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>2. Checking Same Community</h4></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#getting weakly connected edges from graph \n",
    "wcc = list(nx.weakly_connected_components(train_graph))\n",
    "\n",
    "def belongs_to_same_wcc(a,b):\n",
    "    index = []\n",
    "    if train_graph.has_edge(b,a):\n",
    "        return 1\n",
    "    if train_graph.has_edge(a,b):\n",
    "            for i in wcc:\n",
    "                if a in i:\n",
    "                    index= i\n",
    "                    break\n",
    "            if (b in index):\n",
    "                train_graph.remove_edge(a,b)\n",
    "                if compute_shortest_path_length(a,b)==-1:\n",
    "                    train_graph.add_edge(a,b)\n",
    "                    return 0\n",
    "                else:\n",
    "                    train_graph.add_edge(a,b)\n",
    "                    return 1\n",
    "            else:\n",
    "                return 0\n",
    "    else:\n",
    "            for i in wcc:\n",
    "                if a in i:\n",
    "                    index= i\n",
    "                    break\n",
    "            if(b in index):\n",
    "                return 1\n",
    "            else:\n",
    "                return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(belongs_to_same_wcc(858,163554))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(belongs_to_same_wcc(669354,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>3. Adamic/Adar Index</h4></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adamic/Adar measures is defined as inverted sum of degrees of common neighbours for given two vertices.\n",
    "$$A(x,y)=\\sum_{u \\in N(x) \\cap N(y)}\\frac{1}{log(|N(u)|)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#adar_index\n",
    "def calc_adar_in(a,b):\n",
    "    sum = 0\n",
    "    try:\n",
    "        n = list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n",
    "        if len(n)!= 0:\n",
    "            for i in n:\n",
    "                sum = sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n",
    "            return sum\n",
    "        else:\n",
    "            return 0\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(calc_adar_in(6354,1535))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>4. Is Person Was Following Back</h4></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def follows_back(a,b):\n",
    "    if train_graph.has_edge(b,a):\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(follows_back(1,189226))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#print_test_case_\n",
    "print(follows_back(14845,1896))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>5. Katz Centrality</h4></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Katz centrality computes the centrality for a node \n",
    "    based on the centrality of its neighbors. It is a \n",
    "    generalization of the eigenvector centrality. The\n",
    "    Katz centrality for node `i` is\n",
    " \n",
    "$$x_i = \\alpha \\sum_{j} A_{ij} x_j + \\beta,$$\n",
    "where `A` is the adjacency matrix of the graph G \n",
    "with eigenvalues $$\\lambda$$.\n",
    "\n",
    "The parameter $$\\beta$$ controls the initial centrality and \n",
    "\n",
    "$$\\alpha < \\frac{1}{\\lambda_{max}}.$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/katz.p'):\n",
    "    katz = nx.katz.katz_centrality(train_graph, alpha=0.005, beta=1)\n",
    "    pickle.dump(katz,open('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/katz.p','wb'))\n",
    "else:\n",
    "    katz = pickle.load(open('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/katz.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Min 0.0007313532484065916\n",
      "Max 0.003394554981699122\n",
      "Mean 0.0007483800935562018\n"
     ]
    }
   ],
   "source": [
    "print('Min', katz[min(katz, key=katz.get)])\n",
    "print('Max', katz[max(katz, key=katz.get)])\n",
    "print('Mean', float(sum(katz.values()))/len(katz))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0007483800935562018\n"
     ]
    }
   ],
   "source": [
    "mean_katz = float(sum(katz.values()))/len(katz)\n",
    "print(mean_katz)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=green><h4>6. Hits Score</h4></font>\n",
    "\n",
    "The HITS algorithm computes two numbers for a node. Authorities estimates the node value based on the incoming links. Hubs estimates the node value based on outgoing links."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/hits.p'):\n",
    "    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n",
    "    pickle.dump(hits,open('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/hits.p','wb'))\n",
    "else:\n",
    "    hits = pickle.load(open('D:/Applied_Ai/Case Studies/Facebook/My Work/fea_sample/hits.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Min 0.0\n",
      "Max 0.004868653378780953\n",
      "Mean 5.615699699344123e-07\n"
     ]
    }
   ],
   "source": [
    "print('Min', hits[0][min(hits[0], key=hits[0].get)])\n",
    "print('Max', hits[0][max(hits[0], key=hits[0].get)])\n",
    "print('Mean', float(sum(hits[0].values()))/len(hits[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=blue><h3>Featurization</h3></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=gren><h4>Reading Sample Data from Both Train and Test</h4></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "if os.path.isfile('D:/Applied_Ai/Case Studies/Facebook/My Work/after_eda/train_after_eda.csv'):\n",
    "    filename = \"train_after_eda.csv\"\n",
    "    #uncomment this line, if you dont know the lentgh of the file name\n",
    "    #here we have hardcoded the number of lines as 15100030\n",
    "    #n_train = sum(1 for line in open(filename)) #number of records in file (excludes header)\n",
    "    \n",
    "    n_train =  15100028\n",
    "    s = 100000 #desired sample size\n",
    "    \n",
    "    skip_train = sorted(random.sample(range(1, n_train+1), n_train-s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.isfile('D:/Applied_Ai/Case Studies/Facebook/My Work/after_eda/train_after_eda.csv'):\n",
    "    filename = \"test_after_eda.csv\"\n",
    "    #uncomment this line, if you dont know the lentgh of the file name\n",
    "    #here we have hardcoded the number of lines as 3775008\n",
    "    #n_test = sum(1 for line in open(filename)) #number of records in file (excludes header)\n",
    "    \n",
    "    n_test = 3775006\n",
    "    s = 50000 #desired sample size\n",
    "    \n",
    "    skip_test = sorted(random.sample(range(1, n_test+1), n_test-s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NUMBER OF ROWS IN THE TRAIN DATA FILE - 15100028\n",
      "NUMBER OF ROWS WE ARE GOING TO ELIMIATE IN TRAIN DATA ARE 15000028\n",
      "NUMBER OF ROWS IN THE TEST DATA FILE - 3775006\n",
      "NUMBER OF ROWS WE ARE GOING TO ELIMIATE IN TEST DATA ARE 3725006\n"
     ]
    }
   ],
   "source": [
    "print(\"NUMBER OF ROWS IN THE TRAIN DATA FILE -\", n_train)\n",
    "print(\"NUMBER OF ROWS WE ARE GOING TO ELIMIATE IN TRAIN DATA ARE\",len(skip_train))\n",
    "print(\"NUMBER OF ROWS IN THE TEST DATA FILE -\", n_test)\n",
    "print(\"NUMBER OF ROWS WE ARE GOING TO ELIMIATE IN TEST DATA ARE\",len(skip_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SIZE OF TRAIN MATRIX - (100002, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1284805</td>\n",
       "      <td>1252082</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1764008</td>\n",
       "      <td>713641</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>462722</td>\n",
       "      <td>1698321</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>996276</td>\n",
       "      <td>883507</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link\n",
       "0       273084           1505602               1\n",
       "1      1284805           1252082               1\n",
       "2      1764008            713641               1\n",
       "3       462722           1698321               1\n",
       "4       996276            883507               1"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_final = pd.read_csv('D:/Applied_Ai/Case Studies/Facebook/My Work/after_eda/train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\n",
    "train_data_final[\"indicator_link\"] = pd.read_csv('D:/Applied_Ai/Case Studies/Facebook/My Work/train_y.csv', skiprows=skip_train, names=['indicator_link'])\n",
    "\n",
    "print(\"SIZE OF TRAIN MATRIX -\",train_data_final.shape)\n",
    "train_data_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SIZE OF TEST MATRIX - (50002, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>848424</td>\n",
       "      <td>784690</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1549883</td>\n",
       "      <td>966112</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>471540</td>\n",
       "      <td>1086897</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>264958</td>\n",
       "      <td>339685</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1288618</td>\n",
       "      <td>816620</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link\n",
       "0       848424            784690               1\n",
       "1      1549883            966112               1\n",
       "2       471540           1086897               1\n",
       "3       264958            339685               1\n",
       "4      1288618            816620               1"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data_final = pd.read_csv('D:/Applied_Ai/Case Studies/Facebook/My Work/after_eda/test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\n",
    "test_data_final[\"indicator_link\"] = pd.read_csv('D:/Applied_Ai/Case Studies/Facebook/My Work/test_y.csv', skiprows=skip_test, names=['indicator_link'])\n",
    "\n",
    "print(\"SIZE OF TEST MATRIX -\",test_data_final.shape)\n",
    "test_data_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=gren><h4>Adding a Set of Features</h4></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__We will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>jaccard_followers</li>\n",
    "<li>jaccard_followees</li>\n",
    "<li>cosine_followers</li>\n",
    "<li>cosine_followees</li>\n",
    "<li>num_followers_s</li>\n",
    "<li>num_followees_s</li>\n",
    "<li>num_followers_d</li>\n",
    "<li>num_followees_d</li>\n",
    "<li>inter_followers</li>\n",
    "<li>inter_followees</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_features_stage1(final_data):\n",
    "    #calculating no of followers & followees for source and destination\n",
    "    #calculating intersection of followers & followees for source and destination\n",
    "    num_followers_s = [] #number of followers of source\n",
    "    num_followees_s = [] #number of followees of source\n",
    "    num_followers_d = [] #number of followers of destination\n",
    "    num_followees_d = [] #number of followees of destination\n",
    "    inter_followers = [] #number of common followers\n",
    "    inter_followees = [] #number of common followees\n",
    "    \n",
    "    for i,row in final_data.iterrows():\n",
    "        try:\n",
    "            s1 = set(train_graph.predecessors(row['source_node']))\n",
    "            s2 = set(train_graph.successors(row['source_node']))\n",
    "        except:\n",
    "            s1 = set()\n",
    "            s2 = set()\n",
    "        try:\n",
    "            d1 = set(train_graph.predecessors(row['destination_node']))\n",
    "            d2 = set(train_graph.successors(row['destination_node']))\n",
    "        except:\n",
    "            d1 = set()\n",
    "            d2 = set()\n",
    "            \n",
    "        num_followers_s.append(len(s1))\n",
    "        num_followees_s.append(len(s2))\n",
    "\n",
    "        num_followers_d.append(len(d1))\n",
    "        num_followees_d.append(len(d2))\n",
    "\n",
    "        inter_followers.append(len(s1.intersection(d1)))\n",
    "        inter_followees.append(len(s2.intersection(d2)))\n",
    "    \n",
    "    return  num_followers_s, num_followees_s, num_followers_d ,num_followees_d, inter_followers, inter_followees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage1.h5'):\n",
    "    #mapping jaccrd followers to train and test data\n",
    "    train_data_final['jaccard_followers'] = train_data_final.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    test_data_final['jaccard_followers'] = test_data_final.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "    #mapping jaccrd followees to train and test data\n",
    "    train_data_final['jaccard_followees'] = train_data_final.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    test_data_final['jaccard_followees'] = test_data_final.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    #mapping cosine followers to train and test data\n",
    "    train_data_final['cosine_followers'] = train_data_final.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    test_data_final['cosine_followers'] = test_data_final.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping cosine followees to train and test data\n",
    "    train_data_final['cosine_followees'] = train_data_final.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    test_data_final['cosine_followees'] = test_data_final.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    #compute_features_stage1 to train and test data\n",
    "    train_data_final['num_followers_s'], train_data_final['num_followers_d'], \\\n",
    "    train_data_final['num_followees_s'], train_data_final['num_followees_d'], \\\n",
    "    train_data_final['inter_followers'], train_data_final['inter_followees'] = compute_features_stage1(train_data_final)\n",
    "    \n",
    "    test_data_final['num_followers_s'], test_data_final['num_followers_d'], \\\n",
    "    test_data_final['num_followees_s'], test_data_final['num_followees_d'], \\\n",
    "    test_data_final['inter_followers'], test_data_final['inter_followees'] = compute_features_stage1(test_data_final)\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    \n",
    "    hdf = HDFStore('storage_sample_stage1.h5')\n",
    "    hdf.put('train_df', train_data_final, format='table', data_columns=True)\n",
    "    hdf.put('test_df', test_data_final, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "    \n",
    "else:\n",
    "    train_data_final = read_hdf('storage_sample_stage1.h5', 'train_df', mode='r')\n",
    "    test_data_final = read_hdf('storage_sample_stage1.h5', 'test_df', mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=gren><h4>Adding New Set of Features</h4></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__We will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>Adar Index</li>\n",
    "<li>Is following back</li>\n",
    "<li>Belongs to same weakly connect components</li>\n",
    "<li>Shortest path between source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage2.h5'):\n",
    "    #mapping adar index on train and test data\n",
    "    train_data_final['adar_index'] = train_data_final.apply(lambda row: calc_adar_in(row['source_node'], row['destination_node']), axis=1)\n",
    "    \n",
    "    test_data_final['adar_index'] = test_data_final.apply(lambda row: calc_adar_in(row['source_node'], row['destination_node']), axis=1)\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping followback or not on train and test data\n",
    "    train_data_final['follows_back'] = train_data_final.apply(lambda row: follows_back(row['source_node'], row['destination_node']), axis=1)\n",
    "\n",
    "    test_data_final['follows_back'] = test_data_final.apply(lambda row: follows_back(row['source_node'], row['destination_node']), axis=1)\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping same component of wcc or not on train and test data\n",
    "    train_data_final['same_comp'] = train_data_final.apply(lambda row: belongs_to_same_wcc(row['source_node'], row['destination_node']), axis=1)\n",
    "\n",
    "    test_data_final['same_comp'] = test_data_final.apply(lambda row: belongs_to_same_wcc(row['source_node'], row['destination_node']), axis=1)\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping shortest path on train and test data\n",
    "    train_data_final['shortest_path'] = train_data_final.apply(lambda row: compute_shortest_path_length(row['source_node'], row['destination_node']), axis=1)\n",
    "    \n",
    "    test_data_final['shortest_path'] = test_data_final.apply(lambda row: compute_shortest_path_length(row['source_node'], row['destination_node']), axis=1)\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    \n",
    "    hdf = HDFStore('storage_sample_stage2.h5')\n",
    "    hdf.put('train_df', train_data_final, format='table', data_columns=True)\n",
    "    hdf.put('test_df', test_data_final, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "    \n",
    "else:\n",
    "    train_data_final = read_hdf('storage_sample_stage2.h5', 'train_df', mode='r')\n",
    "    test_data_final = read_hdf('storage_sample_stage2.h5', 'test_df', mode='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "      <th>jaccard_followers</th>\n",
       "      <th>jaccard_followees</th>\n",
       "      <th>cosine_followers</th>\n",
       "      <th>cosine_followees</th>\n",
       "      <th>num_followers_s</th>\n",
       "      <th>num_followers_d</th>\n",
       "      <th>num_followees_s</th>\n",
       "      <th>num_followees_d</th>\n",
       "      <th>inter_followers</th>\n",
       "      <th>inter_followees</th>\n",
       "      <th>adar_index</th>\n",
       "      <th>follows_back</th>\n",
       "      <th>same_comp</th>\n",
       "      <th>shortest_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1284805</td>\n",
       "      <td>1252082</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70</td>\n",
       "      <td>222</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1764008</td>\n",
       "      <td>713641</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28</td>\n",
       "      <td>31</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>462722</td>\n",
       "      <td>1698321</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>996276</td>\n",
       "      <td>883507</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019838</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link  jaccard_followers  \\\n",
       "0       273084           1505602               1                  0   \n",
       "1      1284805           1252082               1                  0   \n",
       "2      1764008            713641               1                  0   \n",
       "3       462722           1698321               1                  0   \n",
       "4       996276            883507               1                  0   \n",
       "\n",
       "   jaccard_followees  cosine_followers  cosine_followees  num_followers_s  \\\n",
       "0                  0          0.000000               0.0               11   \n",
       "1                  0          0.000000               0.0               70   \n",
       "2                  0          0.000000               0.0               28   \n",
       "3                  0          0.000000               0.0                1   \n",
       "4                  0          0.019838               0.0               21   \n",
       "\n",
       "   num_followers_d  num_followees_s  num_followees_d  inter_followers  \\\n",
       "0               15                6                8                0   \n",
       "1              222                3                3                0   \n",
       "2               31                9                7                0   \n",
       "3               22               15               16                0   \n",
       "4               24               11                5                1   \n",
       "\n",
       "   inter_followees  adar_index  follows_back  same_comp  shortest_path  \n",
       "0                0         0.0             0          1              4  \n",
       "1                0         0.0             1          1              3  \n",
       "2                0         0.0             0          1              3  \n",
       "3                0         0.0             0          1              3  \n",
       "4                0         0.0             1          1              3  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=gren><h4>Adding New Set of Features</h4></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__We will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>Weight Features\n",
    "    <ul>\n",
    "        <li>Weight of incoming edges</li>\n",
    "        <li>Weight of outgoing edges</li>\n",
    "        <li>Weight of incoming edges + Weight of outgoing edges</li>\n",
    "        <li>Weight of incoming edges * Weight of outgoing edges</li>\n",
    "        <li>2*Weight of incoming edges + Weight of outgoing edges</li>\n",
    "        <li>Weight of incoming edges + 2*Weight of outgoing edges</li>\n",
    "    </ul>\n",
    "</li>\n",
    "<li>Page Ranking of Source</li>\n",
    "<li>Page Ranking of Destination</li>\n",
    "<li>Katz of source</li>\n",
    "<li>Katz of dest</li>\n",
    "<li>Hubs of source</li>\n",
    "<li>Hubs of dest</li>\n",
    "<li>Authorities_s of Source</li>\n",
    "<li>Authorities_s of Destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Weight Features\n",
    "\n",
    "\\begin{equation}\n",
    "W = \\frac{1}{\\sqrt{1+|X|}}\n",
    "\\end{equation}\n",
    "\n",
    "it is directed graph so calculated Weighted in and Weighted out differently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████| 1780722/1780722 [00:13<00:00, 134520.33it/s]\n"
     ]
    }
   ],
   "source": [
    "#weight for source and destination of each link\n",
    "Weight_in = {}\n",
    "Weight_out = {}\n",
    "\n",
    "for i in  tqdm(train_graph.nodes()):\n",
    "    s1 = set(train_graph.predecessors(i))\n",
    "    w_in = 1.0/(np.sqrt(1+len(s1)))\n",
    "    Weight_in[i] = w_in\n",
    "    \n",
    "    s2 = set(train_graph.successors(i))\n",
    "    w_out = 1.0/(np.sqrt(1+len(s2)))\n",
    "    Weight_out[i] = w_out\n",
    "    \n",
    "#for imputing with mean\n",
    "mean_weight_in = np.mean(list(Weight_in.values()))\n",
    "mean_weight_out = np.mean(list(Weight_out.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage3.h5'):\n",
    "    \n",
    "    train_data_final['weight_in'] = train_data_final.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    train_data_final['weight_out'] = train_data_final.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "    test_data_final['weight_in'] = test_data_final.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    test_data_final['weight_out'] = test_data_final.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    train_data_final['weight_f1'] = train_data_final.weight_in + train_data_final.weight_out\n",
    "    train_data_final['weight_f2'] = train_data_final.weight_in * train_data_final.weight_out\n",
    "    train_data_final['weight_f3'] = (2*train_data_final.weight_in + 1*train_data_final.weight_out)\n",
    "    train_data_final['weight_f4'] = (1*train_data_final.weight_in + 2*train_data_final.weight_out)\n",
    "\n",
    "    test_data_final['weight_f1'] = test_data_final.weight_in + test_data_final.weight_out\n",
    "    test_data_final['weight_f2'] = test_data_final.weight_in * test_data_final.weight_out\n",
    "    test_data_final['weight_f3'] = (2*test_data_final.weight_in + 1*test_data_final.weight_out)\n",
    "    test_data_final['weight_f4'] = (1*test_data_final.weight_in + 2*test_data_final.weight_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage3.h5'):\n",
    "    #page rank for source and destination in train and test\n",
    "    #if anything not there in train graph then adding mean page rank \n",
    "    train_data_final['page_rank_s'] = train_data_final.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    train_data_final['page_rank_d'] = train_data_final.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "\n",
    "    test_data_final['page_rank_s'] = test_data_final.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    test_data_final['page_rank_d'] = test_data_final.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    #Katz centrality score for source and destination in train and test\n",
    "    #if anything not there in train graph then adding mean katz score\n",
    "    train_data_final['katz_s'] = train_data_final.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    train_data_final['katz_d'] = train_data_final.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "\n",
    "    test_data_final['katz_s'] = test_data_final.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    test_data_final['katz_d'] = test_data_final.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    #Hits algorithm score for source and destination in train and test\n",
    "    #if anything not there in train graph then adding 0\n",
    "    train_data_final['hubs_s'] = train_data_final.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    train_data_final['hubs_d'] = train_data_final.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "\n",
    "    test_data_final['hubs_s'] = test_data_final.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    test_data_final['hubs_d'] = test_data_final.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    #Hits algorithm score for source and destination in Train and Test\n",
    "    #if anything not there in train graph then adding 0\n",
    "    train_data_final['authorities_s'] = train_data_final.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    train_data_final['authorities_d'] = train_data_final.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "\n",
    "    test_data_final['authorities_s'] = test_data_final.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    test_data_final['authorities_d'] = test_data_final.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "    hdf = HDFStore('storage_sample_stage3.h5')\n",
    "    hdf.put('train_df', train_data_final, format='table', data_columns=True)\n",
    "    hdf.put('test_df', test_data_final, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    train_data_final = read_hdf('storage_sample_stage3.h5', 'train_df', mode='r')\n",
    "    test_data_final = read_hdf('storage_sample_stage3.h5', 'test_df', mode='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "      <th>jaccard_followers</th>\n",
       "      <th>jaccard_followees</th>\n",
       "      <th>cosine_followers</th>\n",
       "      <th>cosine_followees</th>\n",
       "      <th>num_followers_s</th>\n",
       "      <th>num_followers_d</th>\n",
       "      <th>num_followees_s</th>\n",
       "      <th>...</th>\n",
       "      <th>weight_f3</th>\n",
       "      <th>weight_f4</th>\n",
       "      <th>page_rank_s</th>\n",
       "      <th>page_rank_d</th>\n",
       "      <th>katz_s</th>\n",
       "      <th>katz_d</th>\n",
       "      <th>hubs_s</th>\n",
       "      <th>hubs_d</th>\n",
       "      <th>authorities_s</th>\n",
       "      <th>authorities_d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>1.005929</td>\n",
       "      <td>0.877964</td>\n",
       "      <td>2.045290e-06</td>\n",
       "      <td>3.459963e-07</td>\n",
       "      <td>0.000773</td>\n",
       "      <td>0.000756</td>\n",
       "      <td>1.943132e-13</td>\n",
       "      <td>1.941103e-13</td>\n",
       "      <td>9.226339e-16</td>\n",
       "      <td>2.231877e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1284805</td>\n",
       "      <td>1252082</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70</td>\n",
       "      <td>222</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1.066965</td>\n",
       "      <td>0.633930</td>\n",
       "      <td>5.733452e-06</td>\n",
       "      <td>2.202787e-07</td>\n",
       "      <td>0.001007</td>\n",
       "      <td>0.000744</td>\n",
       "      <td>3.821852e-09</td>\n",
       "      <td>7.418056e-16</td>\n",
       "      <td>9.628271e-14</td>\n",
       "      <td>2.688395e-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1764008</td>\n",
       "      <td>713641</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28</td>\n",
       "      <td>31</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0.809232</td>\n",
       "      <td>0.669781</td>\n",
       "      <td>1.785100e-06</td>\n",
       "      <td>1.112847e-06</td>\n",
       "      <td>0.000843</td>\n",
       "      <td>0.000766</td>\n",
       "      <td>2.078025e-13</td>\n",
       "      <td>1.648915e-16</td>\n",
       "      <td>8.932667e-16</td>\n",
       "      <td>3.010892e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>462722</td>\n",
       "      <td>1698321</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>0.708514</td>\n",
       "      <td>0.667029</td>\n",
       "      <td>1.846584e-07</td>\n",
       "      <td>5.703804e-07</td>\n",
       "      <td>0.000735</td>\n",
       "      <td>0.000791</td>\n",
       "      <td>2.444041e-13</td>\n",
       "      <td>4.082758e-15</td>\n",
       "      <td>2.812397e-15</td>\n",
       "      <td>6.476138e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>996276</td>\n",
       "      <td>883507</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019838</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0.777350</td>\n",
       "      <td>0.688675</td>\n",
       "      <td>1.632419e-06</td>\n",
       "      <td>3.324746e-07</td>\n",
       "      <td>0.000816</td>\n",
       "      <td>0.000780</td>\n",
       "      <td>3.141020e-13</td>\n",
       "      <td>7.733657e-15</td>\n",
       "      <td>5.007674e-14</td>\n",
       "      <td>1.023551e-13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link  jaccard_followers  \\\n",
       "0       273084           1505602               1                  0   \n",
       "1      1284805           1252082               1                  0   \n",
       "2      1764008            713641               1                  0   \n",
       "3       462722           1698321               1                  0   \n",
       "4       996276            883507               1                  0   \n",
       "\n",
       "   jaccard_followees  cosine_followers  cosine_followees  num_followers_s  \\\n",
       "0                  0          0.000000               0.0               11   \n",
       "1                  0          0.000000               0.0               70   \n",
       "2                  0          0.000000               0.0               28   \n",
       "3                  0          0.000000               0.0                1   \n",
       "4                  0          0.019838               0.0               21   \n",
       "\n",
       "   num_followers_d  num_followees_s  ...  weight_f3  weight_f4   page_rank_s  \\\n",
       "0               15                6  ...   1.005929   0.877964  2.045290e-06   \n",
       "1              222                3  ...   1.066965   0.633930  5.733452e-06   \n",
       "2               31                9  ...   0.809232   0.669781  1.785100e-06   \n",
       "3               22               15  ...   0.708514   0.667029  1.846584e-07   \n",
       "4               24               11  ...   0.777350   0.688675  1.632419e-06   \n",
       "\n",
       "    page_rank_d    katz_s    katz_d        hubs_s        hubs_d  \\\n",
       "0  3.459963e-07  0.000773  0.000756  1.943132e-13  1.941103e-13   \n",
       "1  2.202787e-07  0.001007  0.000744  3.821852e-09  7.418056e-16   \n",
       "2  1.112847e-06  0.000843  0.000766  2.078025e-13  1.648915e-16   \n",
       "3  5.703804e-07  0.000735  0.000791  2.444041e-13  4.082758e-15   \n",
       "4  3.324746e-07  0.000816  0.000780  3.141020e-13  7.733657e-15   \n",
       "\n",
       "   authorities_s  authorities_d  \n",
       "0   9.226339e-16   2.231877e-15  \n",
       "1   9.628271e-14   2.688395e-11  \n",
       "2   8.932667e-16   3.010892e-15  \n",
       "3   2.812397e-15   6.476138e-14  \n",
       "4   5.007674e-14   1.023551e-13  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=gren><h4>Adding New Set of Features</h4></font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__We will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "- **SVD Features** for both source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ref: https://stats.stackexchange.com/questions/455334/singular-value-decomposition-svd-for-feature-selection\n",
    "#ref: https://analyticsindiamag.com/singular-value-decomposition-svd-application-recommender-system/\n",
    "def svd(x, S):\n",
    "    try:\n",
    "        z = sadj_dict[x]\n",
    "        return S[z]\n",
    "    except:\n",
    "        return [0,0,0,0,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for svd features to get feature vector creating a dict node val and index in svd vector\n",
    "sadj_col = sorted(train_graph.nodes())\n",
    "sadj_dict = { val:idx for idx,val in enumerate(sadj_col)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "Adj = nx.adjacency_matrix(train_graph, nodelist=sorted(train_graph.nodes())).asfptype()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adjacency Matrix Shape (1780722, 1780722)\n",
      "U Shape (1780722, 6)\n",
      "V Shape (6, 1780722)\n",
      "s Shape (6,)\n"
     ]
    }
   ],
   "source": [
    "U, s, V = svds(Adj, k=6)\n",
    "\n",
    "print('Adjacency Matrix Shape', Adj.shape)\n",
    "print('U Shape', U.shape)\n",
    "print('V Shape', V.shape)\n",
    "print('s Shape', s.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage4.h5'):\n",
    "    \n",
    "    train_data_final[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n",
    "    train_data_final.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    \n",
    "    train_data_final[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n",
    "    train_data_final.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    train_data_final[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n",
    "    train_data_final.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "\n",
    "    train_data_final[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n",
    "    train_data_final.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    test_data_final[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n",
    "    test_data_final.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    \n",
    "    test_data_final[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n",
    "    test_data_final.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "    test_data_final[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n",
    "    test_data_final.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "\n",
    "    test_data_final[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n",
    "    test_data_final.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "    #----------------------------------------------------------------------------------------------------------------\n",
    "\n",
    "    hdf = HDFStore('storage_sample_stage4.h5')\n",
    "    hdf.put('train_df',train_data_final, format='table', data_columns=True)\n",
    "    hdf.put('test_df',test_data_final, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "    \n",
    "else:\n",
    "    train_data_final = read_hdf('storage_sample_stage4.h5', 'train_df', mode='r')\n",
    "    test_data_final = read_hdf('storage_sample_stage4.h5', 'test_df', mode='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "      <th>jaccard_followers</th>\n",
       "      <th>jaccard_followees</th>\n",
       "      <th>cosine_followers</th>\n",
       "      <th>cosine_followees</th>\n",
       "      <th>num_followers_s</th>\n",
       "      <th>num_followers_d</th>\n",
       "      <th>num_followees_s</th>\n",
       "      <th>...</th>\n",
       "      <th>svd_v_s_3</th>\n",
       "      <th>svd_v_s_4</th>\n",
       "      <th>svd_v_s_5</th>\n",
       "      <th>svd_v_s_6</th>\n",
       "      <th>svd_v_d_1</th>\n",
       "      <th>svd_v_d_2</th>\n",
       "      <th>svd_v_d_3</th>\n",
       "      <th>svd_v_d_4</th>\n",
       "      <th>svd_v_d_5</th>\n",
       "      <th>svd_v_d_6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>1.983699e-06</td>\n",
       "      <td>1.545081e-13</td>\n",
       "      <td>8.108363e-13</td>\n",
       "      <td>1.719703e-14</td>\n",
       "      <td>-1.355367e-12</td>\n",
       "      <td>4.675311e-13</td>\n",
       "      <td>1.128586e-06</td>\n",
       "      <td>6.616662e-14</td>\n",
       "      <td>9.771073e-13</td>\n",
       "      <td>4.160016e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1284805</td>\n",
       "      <td>1252082</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70</td>\n",
       "      <td>222</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>3.663967e-05</td>\n",
       "      <td>9.669868e-11</td>\n",
       "      <td>1.141963e-09</td>\n",
       "      <td>1.794619e-12</td>\n",
       "      <td>-2.851073e-12</td>\n",
       "      <td>1.971059e-11</td>\n",
       "      <td>3.528475e-07</td>\n",
       "      <td>6.538768e-11</td>\n",
       "      <td>2.260206e-11</td>\n",
       "      <td>5.010919e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1764008</td>\n",
       "      <td>713641</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28</td>\n",
       "      <td>31</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>1.402432e-09</td>\n",
       "      <td>3.158536e-12</td>\n",
       "      <td>9.925285e-12</td>\n",
       "      <td>1.664933e-14</td>\n",
       "      <td>-1.683850e-12</td>\n",
       "      <td>8.949512e-12</td>\n",
       "      <td>6.103515e-11</td>\n",
       "      <td>3.314010e-13</td>\n",
       "      <td>1.082855e-12</td>\n",
       "      <td>5.612008e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>462722</td>\n",
       "      <td>1698321</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>8.378921e-08</td>\n",
       "      <td>5.215216e-14</td>\n",
       "      <td>2.932388e-13</td>\n",
       "      <td>5.242047e-14</td>\n",
       "      <td>-2.869186e-11</td>\n",
       "      <td>6.766469e-12</td>\n",
       "      <td>4.520495e-05</td>\n",
       "      <td>1.912283e-12</td>\n",
       "      <td>2.725392e-10</td>\n",
       "      <td>1.207092e-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>996276</td>\n",
       "      <td>883507</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.019838</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>7.261455e-06</td>\n",
       "      <td>4.465798e-12</td>\n",
       "      <td>1.410598e-09</td>\n",
       "      <td>9.333835e-13</td>\n",
       "      <td>-1.015656e-09</td>\n",
       "      <td>2.191805e-11</td>\n",
       "      <td>3.120173e-05</td>\n",
       "      <td>5.287823e-12</td>\n",
       "      <td>1.664792e-11</td>\n",
       "      <td>1.907803e-12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link  jaccard_followers  \\\n",
       "0       273084           1505602               1                  0   \n",
       "1      1284805           1252082               1                  0   \n",
       "2      1764008            713641               1                  0   \n",
       "3       462722           1698321               1                  0   \n",
       "4       996276            883507               1                  0   \n",
       "\n",
       "   jaccard_followees  cosine_followers  cosine_followees  num_followers_s  \\\n",
       "0                  0          0.000000               0.0               11   \n",
       "1                  0          0.000000               0.0               70   \n",
       "2                  0          0.000000               0.0               28   \n",
       "3                  0          0.000000               0.0                1   \n",
       "4                  0          0.019838               0.0               21   \n",
       "\n",
       "   num_followers_d  num_followees_s  ...     svd_v_s_3     svd_v_s_4  \\\n",
       "0               15                6  ...  1.983699e-06  1.545081e-13   \n",
       "1              222                3  ...  3.663967e-05  9.669868e-11   \n",
       "2               31                9  ...  1.402432e-09  3.158536e-12   \n",
       "3               22               15  ...  8.378921e-08  5.215216e-14   \n",
       "4               24               11  ...  7.261455e-06  4.465798e-12   \n",
       "\n",
       "      svd_v_s_5     svd_v_s_6     svd_v_d_1     svd_v_d_2     svd_v_d_3  \\\n",
       "0  8.108363e-13  1.719703e-14 -1.355367e-12  4.675311e-13  1.128586e-06   \n",
       "1  1.141963e-09  1.794619e-12 -2.851073e-12  1.971059e-11  3.528475e-07   \n",
       "2  9.925285e-12  1.664933e-14 -1.683850e-12  8.949512e-12  6.103515e-11   \n",
       "3  2.932388e-13  5.242047e-14 -2.869186e-11  6.766469e-12  4.520495e-05   \n",
       "4  1.410598e-09  9.333835e-13 -1.015656e-09  2.191805e-11  3.120173e-05   \n",
       "\n",
       "      svd_v_d_4     svd_v_d_5     svd_v_d_6  \n",
       "0  6.616662e-14  9.771073e-13  4.160016e-14  \n",
       "1  6.538768e-11  2.260206e-11  5.010919e-10  \n",
       "2  3.314010e-13  1.082855e-12  5.612008e-14  \n",
       "3  1.912283e-12  2.725392e-10  1.207092e-12  \n",
       "4  5.287823e-12  1.664792e-11  1.907803e-12  \n",
       "\n",
       "[5 rows x 55 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **Preferential Attachement** One well-known concept in social networks is that users with many friends tend to create more connections in the future. This is due to the fact that in some social networks, like in finance, the rich get richer. We estimate how ”rich” our two vertices are by calculating the multiplication between the number of friends (|Γ(x)|) or followers each vertex has. It may be noted that the similarity index does not require any node neighbor information; therefore, this similarity index has the lowest computational complexity.\n",
    "\n",
    "Ref: http://be.amazd.com/link-prediction/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preferential attachement for followers\n",
    "def preferential_followers(df):\n",
    "    nf_s = np.array(df['num_followers_s'])\n",
    "    nf_d = np.array(df['num_followers_d'])\n",
    "    \n",
    "    preferential_followers = []\n",
    "    \n",
    "    for i in range(len(nf_s)):\n",
    "        preferential_followers.append(nf_d[i]*nf_s[i])\n",
    "    df['prefer_attach_followers'] = preferential_followers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preferential attachement for followees\n",
    "def preferential_followees(df):\n",
    "    nf_s = np.array(df['num_followees_s'])\n",
    "    nf_d = np.array(df['num_followees_d'])\n",
    "    \n",
    "    preferential_followees=[]\n",
    "    \n",
    "    for i in range(len(nf_s)):\n",
    "        preferential_followees.append(nf_d[i]*nf_s[i])\n",
    "    df['prefer_attach_followees'] = preferential_followees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mapping preferential_followers on train and test data\n",
    "preferential_followers(train_data_final)\n",
    "preferential_followers(test_data_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mapping preferential_followees on train and test data\n",
    "preferential_followees(train_data_final)\n",
    "preferential_followees(test_data_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "      <th>jaccard_followers</th>\n",
       "      <th>jaccard_followees</th>\n",
       "      <th>cosine_followers</th>\n",
       "      <th>cosine_followees</th>\n",
       "      <th>num_followers_s</th>\n",
       "      <th>num_followers_d</th>\n",
       "      <th>num_followees_s</th>\n",
       "      <th>...</th>\n",
       "      <th>svd_v_s_5</th>\n",
       "      <th>svd_v_s_6</th>\n",
       "      <th>svd_v_d_1</th>\n",
       "      <th>svd_v_d_2</th>\n",
       "      <th>svd_v_d_3</th>\n",
       "      <th>svd_v_d_4</th>\n",
       "      <th>svd_v_d_5</th>\n",
       "      <th>svd_v_d_6</th>\n",
       "      <th>prefer_attach_followers</th>\n",
       "      <th>prefer_attach_followees</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>8.108363e-13</td>\n",
       "      <td>1.719703e-14</td>\n",
       "      <td>-1.355367e-12</td>\n",
       "      <td>4.675311e-13</td>\n",
       "      <td>1.128586e-06</td>\n",
       "      <td>6.616662e-14</td>\n",
       "      <td>9.771073e-13</td>\n",
       "      <td>4.160016e-14</td>\n",
       "      <td>165</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1284805</td>\n",
       "      <td>1252082</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70</td>\n",
       "      <td>222</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1.141963e-09</td>\n",
       "      <td>1.794619e-12</td>\n",
       "      <td>-2.851073e-12</td>\n",
       "      <td>1.971059e-11</td>\n",
       "      <td>3.528475e-07</td>\n",
       "      <td>6.538768e-11</td>\n",
       "      <td>2.260206e-11</td>\n",
       "      <td>5.010919e-10</td>\n",
       "      <td>15540</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link  jaccard_followers  \\\n",
       "0       273084           1505602               1                  0   \n",
       "1      1284805           1252082               1                  0   \n",
       "\n",
       "   jaccard_followees  cosine_followers  cosine_followees  num_followers_s  \\\n",
       "0                  0               0.0               0.0               11   \n",
       "1                  0               0.0               0.0               70   \n",
       "\n",
       "   num_followers_d  num_followees_s  ...     svd_v_s_5     svd_v_s_6  \\\n",
       "0               15                6  ...  8.108363e-13  1.719703e-14   \n",
       "1              222                3  ...  1.141963e-09  1.794619e-12   \n",
       "\n",
       "      svd_v_d_1     svd_v_d_2     svd_v_d_3     svd_v_d_4     svd_v_d_5  \\\n",
       "0 -1.355367e-12  4.675311e-13  1.128586e-06  6.616662e-14  9.771073e-13   \n",
       "1 -2.851073e-12  1.971059e-11  3.528475e-07  6.538768e-11  2.260206e-11   \n",
       "\n",
       "      svd_v_d_6  prefer_attach_followers  prefer_attach_followees  \n",
       "0  4.160016e-14                      165                       48  \n",
       "1  5.010919e-10                    15540                        9  \n",
       "\n",
       "[2 rows x 57 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "      <th>jaccard_followers</th>\n",
       "      <th>jaccard_followees</th>\n",
       "      <th>cosine_followers</th>\n",
       "      <th>cosine_followees</th>\n",
       "      <th>num_followers_s</th>\n",
       "      <th>num_followers_d</th>\n",
       "      <th>num_followees_s</th>\n",
       "      <th>...</th>\n",
       "      <th>svd_v_s_5</th>\n",
       "      <th>svd_v_s_6</th>\n",
       "      <th>svd_v_d_1</th>\n",
       "      <th>svd_v_d_2</th>\n",
       "      <th>svd_v_d_3</th>\n",
       "      <th>svd_v_d_4</th>\n",
       "      <th>svd_v_d_5</th>\n",
       "      <th>svd_v_d_6</th>\n",
       "      <th>prefer_attach_followers</th>\n",
       "      <th>prefer_attach_followees</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>848424</td>\n",
       "      <td>784690</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.029161</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>4.341594e-13</td>\n",
       "      <td>5.535501e-14</td>\n",
       "      <td>-9.994075e-10</td>\n",
       "      <td>5.791910e-10</td>\n",
       "      <td>3.512361e-07</td>\n",
       "      <td>2.486660e-09</td>\n",
       "      <td>2.771127e-09</td>\n",
       "      <td>1.727693e-12</td>\n",
       "      <td>36</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1549883</td>\n",
       "      <td>966112</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.183702</td>\n",
       "      <td>0.800641</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>-5.035064e-19</td>\n",
       "      <td>3.516028e-19</td>\n",
       "      <td>6.344320e-19</td>\n",
       "      <td>2.071155e-18</td>\n",
       "      <td>9.349002e-19</td>\n",
       "      <td>-3.258443e-19</td>\n",
       "      <td>-4.820283e-19</td>\n",
       "      <td>3.265825e-19</td>\n",
       "      <td>144</td>\n",
       "      <td>143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link  jaccard_followers  \\\n",
       "0       848424            784690               1                  0   \n",
       "1      1549883            966112               1                  0   \n",
       "\n",
       "   jaccard_followees  cosine_followers  cosine_followees  num_followers_s  \\\n",
       "0                  0          0.029161          0.000000                6   \n",
       "1                  0          0.183702          0.800641               12   \n",
       "\n",
       "   num_followers_d  num_followees_s  ...     svd_v_s_5     svd_v_s_6  \\\n",
       "0                6               14  ...  4.341594e-13  5.535501e-14   \n",
       "1               12               11  ... -5.035064e-19  3.516028e-19   \n",
       "\n",
       "      svd_v_d_1     svd_v_d_2     svd_v_d_3     svd_v_d_4     svd_v_d_5  \\\n",
       "0 -9.994075e-10  5.791910e-10  3.512361e-07  2.486660e-09  2.771127e-09   \n",
       "1  6.344320e-19  2.071155e-18  9.349002e-19 -3.258443e-19 -4.820283e-19   \n",
       "\n",
       "      svd_v_d_6  prefer_attach_followers  prefer_attach_followees  \n",
       "0  1.727693e-12                       36                      126  \n",
       "1  3.265825e-19                      144                      143  \n",
       "\n",
       "[2 rows x 57 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data_final.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **SVD Dot**\n",
    "\n",
    "SVD Dot is dot product between svd sourse node and svd destination node features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#svd dot for train datasets\n",
    "\n",
    "#source_node\n",
    "u_s1, u_s2, u_s3 = train_data_final['svd_u_s_1'], train_data_final['svd_u_s_2'], train_data_final['svd_u_s_3']\n",
    "u_s4, u_s5, u_s6 = train_data_final['svd_u_s_4'], train_data_final['svd_u_s_5'], train_data_final['svd_u_s_6']\n",
    "\n",
    "v_s1, v_s2, v_s3 = train_data_final['svd_v_s_1'], train_data_final['svd_v_s_2'], train_data_final['svd_v_s_3']\n",
    "v_s4, v_s5, v_s6 = train_data_final['svd_v_s_4'], train_data_final['svd_v_s_5'], train_data_final['svd_v_s_6']\n",
    "#-------------------------------------------------------------------------------------------------------------\n",
    "#destination_node\n",
    "u_d1, u_d2, u_d3 = train_data_final['svd_u_d_1'], train_data_final['svd_u_d_2'], train_data_final['svd_u_d_3']\n",
    "u_d4, u_d5, u_d6 = train_data_final['svd_u_d_4'], train_data_final['svd_u_d_5'], train_data_final['svd_u_d_6']\n",
    "\n",
    "v_d1, v_d2, v_d3 = train_data_final['svd_v_d_1'], train_data_final['svd_v_d_2'], train_data_final['svd_v_d_3']\n",
    "v_d4, v_d5, v_d6 = train_data_final['svd_v_d_4'], train_data_final['svd_v_d_5'], train_data_final['svd_v_d_6']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "svd_dot_train = []\n",
    "\n",
    "for i in range(len(np.array(u_s1))):\n",
    "    a=[]\n",
    "    b=[]\n",
    "    #_u_source_\n",
    "    a.append(np.array(u_s1[i]))\n",
    "    a.append(np.array(u_s2[i]))\n",
    "    a.append(np.array(u_s3[i]))\n",
    "    a.append(np.array(u_s4[i]))\n",
    "    a.append(np.array(u_s5[i]))\n",
    "    a.append(np.array(u_s6[i]))\n",
    "    #_v_source_\n",
    "    a.append(np.array(v_s1[i]))\n",
    "    a.append(np.array(v_s2[i]))\n",
    "    a.append(np.array(v_s3[i]))\n",
    "    a.append(np.array(v_s4[i]))\n",
    "    a.append(np.array(v_s5[i]))\n",
    "    a.append(np.array(v_s6[i]))\n",
    "    #_u_destination_\n",
    "    b.append(np.array(u_d1[i]))\n",
    "    b.append(np.array(u_d2[i]))\n",
    "    b.append(np.array(u_d3[i]))\n",
    "    b.append(np.array(u_d4[i]))\n",
    "    b.append(np.array(u_d5[i]))\n",
    "    b.append(np.array(u_d6[i]))\n",
    "    #_v_destination_\n",
    "    b.append(np.array(v_d1[i]))\n",
    "    b.append(np.array(v_d2[i]))\n",
    "    b.append(np.array(v_d3[i]))\n",
    "    b.append(np.array(v_d4[i]))\n",
    "    b.append(np.array(v_d5[i]))\n",
    "    b.append(np.array(v_d6[i]))\n",
    "    \n",
    "    svd_dot_train.append(np.dot(a,b))\n",
    "    \n",
    "train_data_final['svd_dot'] = svd_dot_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "      <th>jaccard_followers</th>\n",
       "      <th>jaccard_followees</th>\n",
       "      <th>cosine_followers</th>\n",
       "      <th>cosine_followees</th>\n",
       "      <th>num_followers_s</th>\n",
       "      <th>num_followers_d</th>\n",
       "      <th>num_followees_s</th>\n",
       "      <th>...</th>\n",
       "      <th>svd_v_s_6</th>\n",
       "      <th>svd_v_d_1</th>\n",
       "      <th>svd_v_d_2</th>\n",
       "      <th>svd_v_d_3</th>\n",
       "      <th>svd_v_d_4</th>\n",
       "      <th>svd_v_d_5</th>\n",
       "      <th>svd_v_d_6</th>\n",
       "      <th>prefer_attach_followers</th>\n",
       "      <th>prefer_attach_followees</th>\n",
       "      <th>svd_dot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>1.719703e-14</td>\n",
       "      <td>-1.355367e-12</td>\n",
       "      <td>4.675311e-13</td>\n",
       "      <td>1.128586e-06</td>\n",
       "      <td>6.616662e-14</td>\n",
       "      <td>9.771073e-13</td>\n",
       "      <td>4.160016e-14</td>\n",
       "      <td>165</td>\n",
       "      <td>48</td>\n",
       "      <td>1.338829e-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1284805</td>\n",
       "      <td>1252082</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70</td>\n",
       "      <td>222</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1.794619e-12</td>\n",
       "      <td>-2.851073e-12</td>\n",
       "      <td>1.971059e-11</td>\n",
       "      <td>3.528475e-07</td>\n",
       "      <td>6.538768e-11</td>\n",
       "      <td>2.260206e-11</td>\n",
       "      <td>5.010919e-10</td>\n",
       "      <td>15540</td>\n",
       "      <td>9</td>\n",
       "      <td>2.590603e-11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link  jaccard_followers  \\\n",
       "0       273084           1505602               1                  0   \n",
       "1      1284805           1252082               1                  0   \n",
       "\n",
       "   jaccard_followees  cosine_followers  cosine_followees  num_followers_s  \\\n",
       "0                  0               0.0               0.0               11   \n",
       "1                  0               0.0               0.0               70   \n",
       "\n",
       "   num_followers_d  num_followees_s  ...     svd_v_s_6     svd_v_d_1  \\\n",
       "0               15                6  ...  1.719703e-14 -1.355367e-12   \n",
       "1              222                3  ...  1.794619e-12 -2.851073e-12   \n",
       "\n",
       "      svd_v_d_2     svd_v_d_3     svd_v_d_4     svd_v_d_5     svd_v_d_6  \\\n",
       "0  4.675311e-13  1.128586e-06  6.616662e-14  9.771073e-13  4.160016e-14   \n",
       "1  1.971059e-11  3.528475e-07  6.538768e-11  2.260206e-11  5.010919e-10   \n",
       "\n",
       "   prefer_attach_followers  prefer_attach_followees       svd_dot  \n",
       "0                      165                       48  1.338829e-11  \n",
       "1                    15540                        9  2.590603e-11  \n",
       "\n",
       "[2 rows x 58 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#svd dot for test datasets\n",
    "\n",
    "#source_node\n",
    "u_s1, u_s2, u_s3 = test_data_final['svd_u_s_1'], test_data_final['svd_u_s_2'], test_data_final['svd_u_s_3']\n",
    "u_s4, u_s5, u_s6 = test_data_final['svd_u_s_4'], test_data_final['svd_u_s_5'], test_data_final['svd_u_s_6']\n",
    "\n",
    "v_s1, v_s2, v_s3 = test_data_final['svd_v_s_1'], test_data_final['svd_v_s_2'], test_data_final['svd_v_s_3']\n",
    "v_s4, v_s5, v_s6 = test_data_final['svd_v_s_4'], test_data_final['svd_v_s_5'], test_data_final['svd_v_s_6']\n",
    "#-------------------------------------------------------------------------------------------------------------\n",
    "#destination_node\n",
    "u_d1, u_d2, u_d3 = test_data_final['svd_u_d_1'], test_data_final['svd_u_d_2'], test_data_final['svd_u_d_3']\n",
    "u_d4, u_d5, u_d6 = test_data_final['svd_u_d_4'], test_data_final['svd_u_d_5'], test_data_final['svd_u_d_6']\n",
    "\n",
    "v_d1, v_d2, v_d3 = test_data_final['svd_v_d_1'], test_data_final['svd_v_d_2'], test_data_final['svd_v_d_3']\n",
    "v_d4, v_d5, v_d6 = test_data_final['svd_v_d_4'], test_data_final['svd_v_d_5'], test_data_final['svd_v_d_6']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "svd_dot_test = []\n",
    "\n",
    "for i in range(len(np.array(u_s1))):\n",
    "    a=[]\n",
    "    b=[]\n",
    "    #_u_source_\n",
    "    a.append(np.array(u_s1[i]))\n",
    "    a.append(np.array(u_s2[i]))\n",
    "    a.append(np.array(u_s3[i]))\n",
    "    a.append(np.array(u_s4[i]))\n",
    "    a.append(np.array(u_s5[i]))\n",
    "    a.append(np.array(u_s6[i]))\n",
    "    #_v_source_\n",
    "    a.append(np.array(v_s1[i]))\n",
    "    a.append(np.array(v_s2[i]))\n",
    "    a.append(np.array(v_s3[i]))\n",
    "    a.append(np.array(v_s4[i]))\n",
    "    a.append(np.array(v_s5[i]))\n",
    "    a.append(np.array(v_s6[i]))\n",
    "    #_u_destination_\n",
    "    b.append(np.array(u_d1[i]))\n",
    "    b.append(np.array(u_d2[i]))\n",
    "    b.append(np.array(u_d3[i]))\n",
    "    b.append(np.array(u_d4[i]))\n",
    "    b.append(np.array(u_d5[i]))\n",
    "    b.append(np.array(u_d6[i]))\n",
    "    #_v_destination_\n",
    "    b.append(np.array(v_d1[i]))\n",
    "    b.append(np.array(v_d2[i]))\n",
    "    b.append(np.array(v_d3[i]))\n",
    "    b.append(np.array(v_d4[i]))\n",
    "    b.append(np.array(v_d5[i]))\n",
    "    b.append(np.array(v_d6[i]))\n",
    "    \n",
    "    svd_dot_test.append(np.dot(a,b))\n",
    "    \n",
    "test_data_final['svd_dot'] = svd_dot_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "      <th>jaccard_followers</th>\n",
       "      <th>jaccard_followees</th>\n",
       "      <th>cosine_followers</th>\n",
       "      <th>cosine_followees</th>\n",
       "      <th>num_followers_s</th>\n",
       "      <th>num_followers_d</th>\n",
       "      <th>num_followees_s</th>\n",
       "      <th>...</th>\n",
       "      <th>svd_v_s_6</th>\n",
       "      <th>svd_v_d_1</th>\n",
       "      <th>svd_v_d_2</th>\n",
       "      <th>svd_v_d_3</th>\n",
       "      <th>svd_v_d_4</th>\n",
       "      <th>svd_v_d_5</th>\n",
       "      <th>svd_v_d_6</th>\n",
       "      <th>prefer_attach_followers</th>\n",
       "      <th>prefer_attach_followees</th>\n",
       "      <th>svd_dot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>848424</td>\n",
       "      <td>784690</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.029161</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>5.535501e-14</td>\n",
       "      <td>-9.994075e-10</td>\n",
       "      <td>5.791910e-10</td>\n",
       "      <td>3.512361e-07</td>\n",
       "      <td>2.486660e-09</td>\n",
       "      <td>2.771127e-09</td>\n",
       "      <td>1.727693e-12</td>\n",
       "      <td>36</td>\n",
       "      <td>126</td>\n",
       "      <td>2.083224e-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1549883</td>\n",
       "      <td>966112</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.183702</td>\n",
       "      <td>0.800641</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>3.516028e-19</td>\n",
       "      <td>6.344320e-19</td>\n",
       "      <td>2.071155e-18</td>\n",
       "      <td>9.349002e-19</td>\n",
       "      <td>-3.258443e-19</td>\n",
       "      <td>-4.820283e-19</td>\n",
       "      <td>3.265825e-19</td>\n",
       "      <td>144</td>\n",
       "      <td>143</td>\n",
       "      <td>4.861132e-34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link  jaccard_followers  \\\n",
       "0       848424            784690               1                  0   \n",
       "1      1549883            966112               1                  0   \n",
       "\n",
       "   jaccard_followees  cosine_followers  cosine_followees  num_followers_s  \\\n",
       "0                  0          0.029161          0.000000                6   \n",
       "1                  0          0.183702          0.800641               12   \n",
       "\n",
       "   num_followers_d  num_followees_s  ...     svd_v_s_6     svd_v_d_1  \\\n",
       "0                6               14  ...  5.535501e-14 -9.994075e-10   \n",
       "1               12               11  ...  3.516028e-19  6.344320e-19   \n",
       "\n",
       "      svd_v_d_2     svd_v_d_3     svd_v_d_4     svd_v_d_5     svd_v_d_6  \\\n",
       "0  5.791910e-10  3.512361e-07  2.486660e-09  2.771127e-09  1.727693e-12   \n",
       "1  2.071155e-18  9.349002e-19 -3.258443e-19 -4.820283e-19  3.265825e-19   \n",
       "\n",
       "   prefer_attach_followers  prefer_attach_followees       svd_dot  \n",
       "0                       36                      126  2.083224e-17  \n",
       "1                      144                      143  4.861132e-34  \n",
       "\n",
       "[2 rows x 58 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdf = HDFStore('storage_sample_stage5.h5')\n",
    "hdf.put('train_df', train_data_final, format='table', data_columns=True)\n",
    "hdf.put('test_df', test_data_final, format='table', data_columns=True)\n",
    "hdf.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}