experiment2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Experiment: End2End random DNS subdomains DDOS detection\n",
    "Get the Data if needed"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from utils.data import download_url\n",
    "download_url(url='https://ndownloader.figshare.com/files/17905811', target_folder='./data/', filename='dns-requests.csv.gz')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false,
     "name": "#%% md\n"
    }
   },
   "source": [
    "Simulate baseline creation in peacetime and attack on github.com domain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import random\n",
    "from algorithm import DistinctHeavyHitters, HeavyHitters\n",
    "from smart_open import open\n",
    "import uuid\n",
    "dwsHH_baseline = DistinctHeavyHitters(k=10000)\n",
    "dwsHH_detector = DistinctHeavyHitters(k=10000)\n",
    "\n",
    "kHH = HeavyHitters(k=10000)\n",
    "first_run = True\n",
    "num_line_peacetime = 85000000\n",
    "counter = 0\n",
    "\n",
    "def is_peacetime():\n",
    "    return counter < num_line_peacetime\n",
    "\n",
    "white_list_subdomains = None\n",
    "white_list_domains = None\n",
    "top_domains_num = 20\n",
    "confidence_threshold = 0.75\n",
    "victims = ['github.com']\n",
    "max_subdomains_per_attack = 1\n",
    "\n",
    "for line in open('data/dns-requests.csv.gz'):\n",
    "    if first_run:\n",
    "        first_run = False\n",
    "        continue\n",
    "    values = line.rstrip().split(',')\n",
    "    counter += 1\n",
    "    if(len(values)==2):\n",
    "        subdomain, domain = values\n",
    "        if is_peacetime():\n",
    "            dwsHH_baseline.update(subdomain, domain)\n",
    "            kHH.update(subdomain)\n",
    "        else:\n",
    "            if white_list_subdomains is None:\n",
    "                white_list_subdomains = kHH.get()\n",
    "                white_list_domains = []\n",
    "                top_domains_list = sorted(dwsHH_baseline.count(), key=lambda x: x[1], reverse=True)\n",
    "                for top_domain_index in range(top_domains_num):\n",
    "                    top_domain = top_domains_list[top_domain_index]\n",
    "                    if top_domain[2] > confidence_threshold:\n",
    "                        white_list_domains.append(top_domain[0])   \n",
    "            dns_stream = []\n",
    "            # choose to inject subdomains random attack on selected domains or not\n",
    "            if random.uniform(0, 1) < 0.2:\n",
    "                # inject attack\n",
    "                # choose attacked domain at random\n",
    "                domain_index = random.randint(0,len(victims)-1)\n",
    "                # choose how many generated subdomains for this attack\n",
    "                dns_stream.append((str(uuid.uuid4()), victims[domain_index]))\n",
    "            # append the original dns request\n",
    "            dns_stream.append((subdomain, domain))\n",
    "            for (subdomain, domain) in dns_stream:  \n",
    "                if subdomain in white_list_subdomains:\n",
    "                    continue\n",
    "                if domain in white_list_domains:\n",
    "                    continue\n",
    "                dwsHH_detector.update(subdomain, domain)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "vip-skyrockcom-lbs\nwww\nusername\nblogspot.l\nq50\nvip-skyrockmobi-lbs\nclb\npages-wildcard\ndq33tynpwunh\nwww3.l\ne8e03d8f-fce6-47db-a48a-1982558191aa\nhttp\nnb-69-164-223-52.newark.nodebalancer\nurlforward\nmx2\np2620\na-us00\nbitrixssd\ns\nweb\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "for i in range(20):\n",
    "    print(white_list_subdomains[i])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "['blogspot.com',\n 'wordpress.com',\n 'skyrock.com',\n 'home.blog',\n 'herokuapp.com',\n 'deviantart.com',\n 'fc2.com',\n 'livejournal.com',\n 'weebly.com',\n 'filetransit.com',\n 'lofter.com',\n 'wixsite.com',\n 'soft112.com',\n 'list-manage.com',\n 'amazonaws.com',\n 'jimdo.com',\n 'wix.com',\n 'azurewebsites.net',\n 'uptodown.com',\n 'blogspot.ca']"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 4
    }
   ],
   "source": [
    "white_list_domains"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "baseline for github.com: 21354 (how many distinct subdomain in peacetime)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "github_baseline = 1\n",
    "for domain in dwsHH_baseline.count():\n",
    "    if domain[0] == 'github.com':\n",
    "        github_baseline = domain[1]\n",
    "print(\"Baseline for github.com: %d (how many distinct subdomain in peacetime)\" % github_baseline)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "github.com 16243296.342877353\nmyshopify.com 384536.65077588946\ncloudflare.net 227763.7876032608\nfood.blog 137231.1965133412\nvideo.blog 117915.23354495593\naltervista.org 107703.70762908598\nart.blog 106280.71926067249\nwindows.net 101418.59947917162\ngooglehosted.com 92433.17745677632\nbokhra.com 86927.75309414702\nbigcartel.com 78017.22972080091\nvigbo.com 70129.20200206093\ncloudfront.net 69248.63179016161\nbjbqz.com 62986.591890642936\nfashion.blog 62423.808337435396\n163.com 57673.533326227516\nschool.blog 55786.85505379681\ngwstest.net 54586.5221201435\nwdstq.com 54584.348984069635\nherokudns.com 54541.16363671564\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "top_domain_in_attack_time = sorted(dwsHH_detector.count(), key=lambda x: x[1], reverse=True)\n",
    "for i in range(20):\n",
    "    domain = top_domain_in_attack_time[i]\n",
    "    print(domain[0],domain[1])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Ratio between github in attack time to baseline: 760.640724\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "github_attacktime= 0\n",
    "for domain in dwsHH_detector.count():\n",
    "    if domain[0] == 'github.com':\n",
    "        github_attacktime = domain[1]\n",
    "github_ratio = github_attacktime / github_baseline\n",
    "print(\"Ratio between github in attack time to baseline: %f\" % (github_attacktime / github_baseline))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Baseline for myshopify.com: 3710 (how many distinct subdomain in peacetime)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "# for compression lets check the second score  \n",
    "myshopify_baseline = 1\n",
    "for domain in dwsHH_baseline.count():\n",
    "    if domain[0] == 'myshopify.com':\n",
    "        myshopify_baseline = domain[1]\n",
    "print(\"Baseline for myshopify.com: %d (how many distinct subdomain in peacetime)\" % myshopify_baseline)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Ratio between myshopify in attack time to baseline: 103.648583\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "myshopify_attacktime= 0\n",
    "for domain in dwsHH_detector.count():\n",
    "    if domain[0] == 'myshopify.com':\n",
    "        myshopify_attacktime = domain[1]\n",
    "myshopify_ratio = myshopify_attacktime / myshopify_baseline\n",
    "print(\"Ratio between myshopify in attack time to baseline: %f\" % (myshopify_attacktime / myshopify_baseline))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}