Exploratory_data_analysis_with_pySpark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DataFrame \n",
    "\n",
    "- What is spark dataframe?\n",
    "\n",
    "- Why is it different than the RDDs?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the spark on jupyter notebook\n",
    "\n",
    "import findspark\n",
    "findspark.init()\n",
    "\n",
    "\n",
    "# To create the dataframe in pyspark \n",
    "\n",
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initializing the sparksesstion \n",
    "\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName(\"Python Spark SQL basic example\") \\\n",
    "    .config(\"spark.some.config.option\", \"some-value\") \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading the csv file in pyspark \n",
    "\n",
    "df = spark.read.csv('country_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-----------+----+-----------------+\n",
      "| _c0|        _c1| _c2|              _c3|\n",
      "+----+-----------+----+-----------------+\n",
      "|null|    Country|Year|              GDP|\n",
      "|   0|Afghanistan|2015|584.2592099999999|\n",
      "|   1|Afghanistan|2014|       612.696514|\n",
      "|   2|Afghanistan|2013|       631.744976|\n",
      "|   3|Afghanistan|2012|          669.959|\n",
      "+----+-----------+----+-----------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Show shows the first few entries in the dataframe \n",
    "# Notice that the column names are not good \n",
    "\n",
    "df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _c0: string (nullable = true)\n",
      " |-- _c1: string (nullable = true)\n",
      " |-- _c2: string (nullable = true)\n",
      " |-- _c3: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# What objects the dataframe contains?\n",
    "# Year and GDP are string. Sometimes they may cause problem. \n",
    "\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.types import StructType, StructField, StringType, IntegerType, FloatType\n",
    "\n",
    "schema = StructType([StructField('_c0', IntegerType(), True), \n",
    "                      StructField('Country', StringType(), True), \n",
    "                      StructField('Year', IntegerType(), True), \n",
    "                      StructField('GDP', FloatType(), True)\n",
    "                     ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------+----+---------+\n",
      "|_c0|    Country|Year|      GDP|\n",
      "+---+-----------+----+---------+\n",
      "|  0|Afghanistan|2015| 584.2592|\n",
      "|  1|Afghanistan|2014|612.69653|\n",
      "|  2|Afghanistan|2013|  631.745|\n",
      "|  3|Afghanistan|2012|  669.959|\n",
      "|  4|Afghanistan|2011| 63.53723|\n",
      "+---+-----------+----+---------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Redefining the dataframe with correct column labels \n",
    "\n",
    "df = spark.read.csv('country_data.csv', header=True, schema=schema)\n",
    "df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _c0: integer (nullable = true)\n",
      " |-- Country: string (nullable = true)\n",
      " |-- Year: integer (nullable = true)\n",
      " |-- GDP: float (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----------+----------+----------+\n",
      "|new_name_1| new_name_2|new_name_3|new_name_4|\n",
      "+----------+-----------+----------+----------+\n",
      "|         0|Afghanistan|      2015|  584.2592|\n",
      "|         1|Afghanistan|      2014| 612.69653|\n",
      "|         2|Afghanistan|      2013|   631.745|\n",
      "|         3|Afghanistan|      2012|   669.959|\n",
      "|         4|Afghanistan|      2011|  63.53723|\n",
      "+----------+-----------+----------+----------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# To rename the columns \n",
    "\n",
    "df.toDF('new_name_1', 'new_name_2', 'new_name_3', 'new_name_4').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows: \t 2938\n",
      "Number of columns: \t 4\n",
      "\n",
      "Name of columns: \n",
      " ['_c0', 'Country', 'Year', 'GDP']\n"
     ]
    }
   ],
   "source": [
    "# To count the total rows in the dataframe.\n",
    "\n",
    "print('Number of rows: \\t', df.count())\n",
    "\n",
    "\n",
    "# Number of columns can be counted in this way\n",
    "\n",
    "print('Number of columns: \\t', len(df.columns))\n",
    "\n",
    "\n",
    "# To view the names of the columns\n",
    "\n",
    "print('\\nName of columns: \\n', df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(_c0=0, Country='Afghanistan', Year=2015, GDP=584.2592163085938),\n",
       " Row(_c0=1, Country='Afghanistan', Year=2014, GDP=612.696533203125),\n",
       " Row(_c0=2, Country='Afghanistan', Year=2013, GDP=631.7449951171875),\n",
       " Row(_c0=3, Country='Afghanistan', Year=2012, GDP=669.958984375),\n",
       " Row(_c0=4, Country='Afghanistan', Year=2011, GDP=63.5372314453125)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# In pandas head() would do similar thing. It is slighly different here. \n",
    "# This method is not recommended for the large dataset. \n",
    "\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(_c0=0, Country='Afghanistan', Year=2015, GDP=584.2592163085938),\n",
       " Row(_c0=1, Country='Afghanistan', Year=2014, GDP=612.696533203125),\n",
       " Row(_c0=2, Country='Afghanistan', Year=2013, GDP=631.7449951171875)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# There is also take doing the similar thing \n",
    "\n",
    "df.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+\n",
      "|    Country|\n",
      "+-----------+\n",
      "|Afghanistan|\n",
      "|Afghanistan|\n",
      "|Afghanistan|\n",
      "|Afghanistan|\n",
      "|Afghanistan|\n",
      "+-----------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Selecting a column from the dataframe \n",
    "\n",
    "df.select('Country').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+---------+\n",
      "|    Country|      GDP|\n",
      "+-----------+---------+\n",
      "|Afghanistan| 584.2592|\n",
      "|Afghanistan|612.69653|\n",
      "|Afghanistan|  631.745|\n",
      "|Afghanistan|  669.959|\n",
      "|Afghanistan| 63.53723|\n",
      "+-----------+---------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Selecting two columns from the dataframe \n",
    "\n",
    "df.select('Country', 'GDP').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+----+---------+\n",
      "|    Country|Year|      GDP|\n",
      "+-----------+----+---------+\n",
      "|Afghanistan|2015| 584.2592|\n",
      "|Afghanistan|2014|612.69653|\n",
      "|Afghanistan|2013|  631.745|\n",
      "|Afghanistan|2012|  669.959|\n",
      "|Afghanistan|2011| 63.53723|\n",
      "+-----------+----+---------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Droping unwanted row from datafrmae\n",
    "\n",
    "df.drop('_c0').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_c0</th>\n",
       "      <th>Country</th>\n",
       "      <th>Year</th>\n",
       "      <th>GDP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2015</td>\n",
       "      <td>584.259216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2014</td>\n",
       "      <td>612.696533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2013</td>\n",
       "      <td>631.744995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2012</td>\n",
       "      <td>669.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2011</td>\n",
       "      <td>63.537231</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   _c0      Country  Year         GDP\n",
       "0    0  Afghanistan  2015  584.259216\n",
       "1    1  Afghanistan  2014  612.696533\n",
       "2    2  Afghanistan  2013  631.744995\n",
       "3    3  Afghanistan  2012  669.958984\n",
       "4    4  Afghanistan  2011   63.537231"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert spark dataframe to Pandas dataframe\n",
    "\n",
    "df.toPandas().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_c0</th>\n",
       "      <th>Country</th>\n",
       "      <th>Year</th>\n",
       "      <th>GDP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2015</td>\n",
       "      <td>584.259216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2014</td>\n",
       "      <td>612.696533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2013</td>\n",
       "      <td>631.744995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2012</td>\n",
       "      <td>669.958984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>2011</td>\n",
       "      <td>63.537231</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   _c0      Country  Year         GDP\n",
       "0    0  Afghanistan  2015  584.259216\n",
       "1    1  Afghanistan  2014  612.696533\n",
       "2    2  Afghanistan  2013  631.744995\n",
       "3    3  Afghanistan  2012  669.958984\n",
       "4    4  Afghanistan  2011   63.537231"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Conversion of the whole dataframe in pandas can be costly. \n",
    "# If the interest is to only look at the head we can do the following. \n",
    "\n",
    "df.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0--------------\n",
      " _c0     | 0           \n",
      " Country | Afghanistan \n",
      " Year    | 2015        \n",
      " GDP     | 584.2592    \n",
      "-RECORD 1--------------\n",
      " _c0     | 1           \n",
      " Country | Afghanistan \n",
      " Year    | 2014        \n",
      " GDP     | 612.69653   \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# It is possibel to view dataframe vertically as well\n",
    "\n",
    "df.show(n=2, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----------+----+---------+\n",
      "| _c0|   Country|Year|      GDP|\n",
      "+----+----------+----+---------+\n",
      "|1539|Luxembourg|2014|119172.74|\n",
      "|1542|Luxembourg|2011|115761.58|\n",
      "|1545|Luxembourg|2008|114293.84|\n",
      "|1540|Luxembourg|2013|113751.85|\n",
      "|1547|Luxembourg|2006| 89739.71|\n",
      "+----+----------+----+---------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# By default the data is orderd by country. \n",
    "# Here is a method to order by different column. \n",
    "\n",
    "df.orderBy('GDP', ascending=False).show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------+-----------+------------------+------------------+\n",
      "|summary|              _c0|    Country|              Year|               GDP|\n",
      "+-------+-----------------+-----------+------------------+------------------+\n",
      "|  count|             2938|       2938|              2938|              2490|\n",
      "|   mean|           1468.5|       null|2007.5187202178352| 7483.158462069025|\n",
      "| stddev|848.2718707269896|       null| 4.613840940258099|14270.169327273006|\n",
      "|    min|                0|Afghanistan|              2000|           1.68135|\n",
      "|    max|             2937|   Zimbabwe|              2015|         119172.74|\n",
      "+-------+-----------------+-----------+------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Descriptive statistics of the datafrmae\n",
    "\n",
    "df.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+------------------+\n",
      "|summary|              Year|               GDP|\n",
      "+-------+------------------+------------------+\n",
      "|  count|              2938|              2490|\n",
      "|   mean|2007.5187202178352| 7483.158462069025|\n",
      "| stddev| 4.613840940258099|14270.169327273006|\n",
      "|    min|              2000|           1.68135|\n",
      "|    max|              2015|         119172.74|\n",
      "+-------+------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Descriptive statistics of only selected columns\n",
    "\n",
    "df.describe('Year', 'GDP').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+\n",
      "|min(Year)|\n",
      "+---------+\n",
      "|     2000|\n",
      "+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# If we only need one descriptive measure this is the way \n",
    "\n",
    "df.agg({'Year':'min'}).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------+\n",
      "|         avg(GDP)|\n",
      "+-----------------+\n",
      "|7483.158462069025|\n",
      "+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# If we want two statistical measure simultaneously we can not \n",
    "# do it in this way. We will see how to do that sortly. \n",
    "\n",
    "df.agg({'GDP':'mean'}).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------+-----------+------------------+------------------+\n",
      "|summary|              _c0|    Country|              Year|               GDP|\n",
      "+-------+-----------------+-----------+------------------+------------------+\n",
      "|  count|             2938|       2938|              2938|              2490|\n",
      "|   mean|           1468.5|       null|2007.5187202178352| 7483.158462069025|\n",
      "| stddev|848.2718707269896|       null| 4.613840940258099|14270.169327273006|\n",
      "|    min|                0|Afghanistan|              2000|           1.68135|\n",
      "|    25%|              734|       null|              2004|         463.85263|\n",
      "|    50%|             1468|       null|              2008|         1764.9739|\n",
      "|    75%|             2203|       null|              2012|          5918.199|\n",
      "|    max|             2937|   Zimbabwe|              2015|         119172.74|\n",
      "+-------+-----------------+-----------+------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# summary() is similar to the describe() but it is more versatile \n",
    "# and it has more functionality as we see below. \n",
    "\n",
    "df.summary().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------+-------+------------------+-----------------+\n",
      "|summary|   _c0|Country|              Year|              GDP|\n",
      "+-------+------+-------+------------------+-----------------+\n",
      "|   mean|1468.5|   null|2007.5187202178352|7483.158462069025|\n",
      "|    50%|  1468|   null|              2008|        1764.9739|\n",
      "+-------+------+-------+------------------+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can choose mean and median only. \n",
    "\n",
    "df.summary('mean', '50%').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+------------------+\n",
      "|summary|               GDP|              Year|\n",
      "+-------+------------------+------------------+\n",
      "|  count|              2490|              2938|\n",
      "|   mean| 7483.158462069025|2007.5187202178352|\n",
      "| stddev|14270.169327273006| 4.613840940258099|\n",
      "|    min|           1.68135|              2000|\n",
      "|    25%|         463.85263|              2004|\n",
      "|    50%|         1764.9739|              2008|\n",
      "|    75%|          5918.199|              2012|\n",
      "|    max|         119172.74|              2015|\n",
      "+-------+------------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can choose the column which we want to analyze \n",
    "\n",
    "df.select('GDP', 'Year').summary().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|               GDP|\n",
      "+-------+------------------+\n",
      "|   mean| 7483.158462069025|\n",
      "| stddev|14270.169327273006|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# And finally we can choose both columns and measures \n",
    "\n",
    "df.select('GDP').summary('mean', 'stddev').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------+------------------+\n",
      "|         avg(GDP)|  stddev_samp(GDP)|\n",
      "+-----------------+------------------+\n",
      "|7483.158462069025|14270.169327273006|\n",
      "+-----------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Here is another way to achieve the same thing. \n",
    "# But summary() is way to go because of its simplicity. \n",
    "\n",
    "from pyspark.sql.functions import avg, stddev\n",
    "\n",
    "df.agg(avg('GDP'), stddev('GDP')).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[452.46319580078125, 1757.177978515625, 5734.443359375]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is to calculate the percentiles from the dataframe\n",
    "# First entry is the column name from which these calculations are made. \n",
    "# Second is the list of percentiles we calculate. 0.5 means 50th and so on. \n",
    "# Last entry means accuracy. 0.01 means accurate upto two places in decimal. \n",
    "\n",
    "df.approxQuantile('GDP', [0.25, 0.5, 0.75], 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|       GDP_freqItems|\n",
      "+--------------------+\n",
      "|[39439.82, 36161....|\n",
      "+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# This calculates the frequency and its repeatition \n",
    "\n",
    "df.freqItems(['GDP'], 0.01).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09047997064705468"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This claculates the correlation between two columns\n",
    "\n",
    "df.corr('GDP', 'Year')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+--------+----+---------+\n",
      "|_c0| Country|Year|      GDP|\n",
      "+---+--------+----+---------+\n",
      "| 21| Albania|2010|494.35883|\n",
      "| 44| Algeria|2003|294.33557|\n",
      "|191| Bahrain|2000|13636.347|\n",
      "|217|Barbados|2006| 15646.56|\n",
      "+---+--------+----+---------+\n",
      "only showing top 4 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# To take random sample from the dataframe. \n",
    "# The first boolean is for weather or not to make replacement in smapling.\n",
    "# Second flooat is to indicate what fraction of populations to take in sample. \n",
    "\n",
    "df_sample = df.sample(False, 0.01)\n",
    "df_sample.show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2938\n",
      "37\n"
     ]
    }
   ],
   "source": [
    "# We see that df_sample is 0.01 times df as indicated above.  \n",
    "\n",
    "print(df.count())\n",
    "print(df_sample.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.groupBy('Country').pivot('Year').sum().show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|             Country|\n",
      "+--------------------+\n",
      "|       Côte d'Ivoire|\n",
      "|                Chad|\n",
      "|Micronesia (Feder...|\n",
      "|            Paraguay|\n",
      "|               Yemen|\n",
      "|             Senegal|\n",
      "|          Cabo Verde|\n",
      "|              Sweden|\n",
      "|            Kiribati|\n",
      "|   Republic of Korea|\n",
      "+--------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# distinct() removes the repeatition in the selection \n",
    "\n",
    "countries = df.select('Country').distinct()\n",
    "countries.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+\n",
      "|            Country|\n",
      "+-------------------+\n",
      "|        Afghanistan|\n",
      "|            Albania|\n",
      "|            Algeria|\n",
      "|             Angola|\n",
      "|Antigua and Barbuda|\n",
      "|          Argentina|\n",
      "|            Armenia|\n",
      "|          Australia|\n",
      "|            Austria|\n",
      "|         Azerbaijan|\n",
      "+-------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can sort the dataframe by selected column name \n",
    "\n",
    "countries.sort('Country').show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "193"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we can count the total number of countries \n",
    "\n",
    "countries.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|             Country|count|\n",
      "+--------------------+-----+\n",
      "|       Côte d'Ivoire|   16|\n",
      "|                Chad|   16|\n",
      "|Micronesia (Feder...|   16|\n",
      "|            Paraguay|   16|\n",
      "+--------------------+-----+\n",
      "only showing top 4 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can group the dataframe by entries of the selected columns \n",
    "# Usually we want to perfrom some descriptive measurement after \n",
    "# grouping them together. \n",
    "\n",
    "df.groupBy('Country').count().show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------------------+---------+\n",
      "|             Country|          avg(GDP)|avg(Year)|\n",
      "+--------------------+------------------+---------+\n",
      "|       Côte d'Ivoire|              null|   2007.5|\n",
      "|                Chad| 484.7916989326477|   2007.5|\n",
      "|Micronesia (Feder...|              null|   2007.5|\n",
      "|            Paraguay|1983.4043321609497|   2007.5|\n",
      "|               Yemen|              null|   2007.5|\n",
      "+--------------------+------------------+---------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can do multiple measurement after grouping \n",
    "\n",
    "df.groupBy('Country').mean('GDP', 'Year').show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----+-----+\n",
      "|   Country|Year|count|\n",
      "+----------+----+-----+\n",
      "| Argentina|2010|    1|\n",
      "|Azerbaijan|2012|    1|\n",
      "|   Bahrain|2014|    1|\n",
      "|  Colombia|2014|    1|\n",
      "+----------+----+-----+\n",
      "only showing top 4 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can also group the dataframe by multiple columns \n",
    "\n",
    "df.groupBy('Country', 'Year').count().show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-------+----+---------+\n",
      "| _c0|Country|Year|      GDP|\n",
      "+----+-------+----+---------+\n",
      "|1813|  Nepal|2015| 743.7653|\n",
      "|1814|  Nepal|2014|  76.2387|\n",
      "|1815|  Nepal|2013|688.61725|\n",
      "|1816|  Nepal|2012| 681.7926|\n",
      "|1817|  Nepal|2011| 692.1167|\n",
      "|1818|  Nepal|2010|592.18353|\n",
      "|1819|  Nepal|2009|  48.7299|\n",
      "|1820|  Nepal|2008|473.84445|\n",
      "|1821|  Nepal|2007|393.88434|\n",
      "|1822|  Nepal|2006|348.63144|\n",
      "|1823|  Nepal|2005|317.89197|\n",
      "|1824|  Nepal|2004| 287.4156|\n",
      "|1825|  Nepal|2003|253.72412|\n",
      "|1826|  Nepal|2002|246.37563|\n",
      "|1827|  Nepal|2001|248.61835|\n",
      "|1828|  Nepal|2000|231.42554|\n",
      "+----+-------+----+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Filter is to make selection from the dataframe\n",
    "\n",
    "df.filter(df['Country']=='Nepal').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+----------+----+---------+\n",
      "| _c0|   Country|Year|      GDP|\n",
      "+----+----------+----+---------+\n",
      "|2120|   Romania|2014| 12.27733|\n",
      "|2253|   Senegal|2011| 18.25321|\n",
      "|2555|Tajikistan|2013|14.214412|\n",
      "+----+----------+----+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Here is another selection using filter with two conditions. \n",
    "\n",
    "df.filter((df['GDP'] < 20) & (df['Year']>2010)).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+-----------+------+------------------+\n",
      "|summary|               _c0|    Country|  Year|               GDP|\n",
      "+-------+------------------+-----------+------+------------------+\n",
      "|  count|               183|        183|   183|               154|\n",
      "|   mean|1475.0273224043715|       null|2000.0|4708.5152019757725|\n",
      "| stddev| 850.9962321664372|       null|   0.0|  9181.57371052946|\n",
      "|    min|                15|Afghanistan|  2000|          3.685949|\n",
      "|    max|              2937|   Zimbabwe|  2000|         48735.996|\n",
      "+-------+------------------+-----------+------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can use descriptive measure on the filtered data\n",
    "\n",
    "df.filter(df['Year']==2000).describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|               GDP|\n",
      "+-------+------------------+\n",
      "|  count|               154|\n",
      "|   mean|4708.5152019757725|\n",
      "| stddev|  9181.57371052946|\n",
      "|    min|          3.685949|\n",
      "|    25%|         263.11246|\n",
      "|    50%|         796.79376|\n",
      "|    75%|         3364.4238|\n",
      "|    max|         48735.996|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Here is a more complex one\n",
    "\n",
    "df.filter(df['Year']==2000).select('GDP').summary().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|               GDP|\n",
      "+-------+------------------+\n",
      "|   mean|4708.5152019757725|\n",
      "| stddev|  9181.57371052946|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Even more complex \n",
    "\n",
    "df.filter(df['Year']==2000).select('GDP').summary('mean', 'stddev').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+---+-------+----+---+\n",
      "|summary|_c0|Country|Year|GDP|\n",
      "+-------+---+-------+----+---+\n",
      "|  count|183|    183| 183|154|\n",
      "+-------+---+-------+----+---+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# where is same as filter \n",
    "\n",
    "df.where(df['Year']==2000).summary('count').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+---------------------+\n",
      "|    Country|((Year - 2000) / 100)|\n",
      "+-----------+---------------------+\n",
      "|Afghanistan|                 0.15|\n",
      "|Afghanistan|                 0.14|\n",
      "|Afghanistan|                 0.13|\n",
      "|Afghanistan|                 0.12|\n",
      "|Afghanistan|                 0.11|\n",
      "+-----------+---------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can do simple mathematics on the column of the dataframe \n",
    "\n",
    "df.select('Country', (df['Year'] - 2000)/100).show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+------------------+\n",
      "|    Country|      (Year / GDP)|\n",
      "+-----------+------------------+\n",
      "|Afghanistan|3.4488116639921658|\n",
      "|Afghanistan|3.2871085290314612|\n",
      "|Afghanistan|3.1864122637435255|\n",
      "|Afghanistan| 3.003168920672033|\n",
      "+-----------+------------------+\n",
      "only showing top 4 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# We can also do operations between two columns \n",
    "\n",
    "df.select('Country', df['Year']/df['GDP']).show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------+----+---------+------------------+\n",
      "|_c0|    Country|Year|      GDP|        new_column|\n",
      "+---+-----------+----+---------+------------------+\n",
      "|  0|Afghanistan|2015| 584.2592|3.4488116639921658|\n",
      "|  1|Afghanistan|2014|612.69653|3.2871085290314612|\n",
      "|  2|Afghanistan|2013|  631.745|3.1864122637435255|\n",
      "|  3|Afghanistan|2012|  669.959| 3.003168920672033|\n",
      "+---+-----------+----+---------+------------------+\n",
      "only showing top 4 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# If we want to create new column with these operations \n",
    "\n",
    "df.withColumn('new_column', df['Year']/df['GDP']).show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Udf provides an option to apply a function in the dataframe.\n",
    "\n",
    "from pyspark.sql.functions import udf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First we create a function \n",
    "\n",
    "def is_developed(gdp):\n",
    "    if gdp < 450:\n",
    "        return 'Undeveloped'\n",
    "    elif 450 < gdp < 6000:\n",
    "        return 'Developing'\n",
    "    else:\n",
    "        return 'Developed'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------+----+---------+-------------+\n",
      "|_c0|    Country|Year|      GDP|is_developed?|\n",
      "+---+-----------+----+---------+-------------+\n",
      "|  0|Afghanistan|2015| 584.2592|   Developing|\n",
      "|  1|Afghanistan|2014|612.69653|   Developing|\n",
      "|  2|Afghanistan|2013|  631.745|   Developing|\n",
      "|  3|Afghanistan|2012|  669.959|   Developing|\n",
      "|  4|Afghanistan|2011| 63.53723|  Undeveloped|\n",
      "+---+-----------+----+---------+-------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Now we create udf using the function we defined \n",
    "\n",
    "poor_udf = udf(is_developed, StringType())\n",
    "\n",
    "\n",
    "# We create a new column using the udf we just defined \n",
    "\n",
    "df.withColumn('is_developed?', poor_udf(df['GDP'])).show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------+----+---------+---------+\n",
      "|_c0|    Country|Year|      GDP|how_long?|\n",
      "+---+-----------+----+---------+---------+\n",
      "|  0|Afghanistan|2015| 584.2592|   Recent|\n",
      "|  1|Afghanistan|2014|612.69653|   Recent|\n",
      "|  2|Afghanistan|2013|  631.745|   Recent|\n",
      "|  3|Afghanistan|2012|  669.959| Long ago|\n",
      "|  4|Afghanistan|2011| 63.53723| Long ago|\n",
      "+---+-----------+----+---------+---------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def how_long(year):\n",
    "    if year < 2013:\n",
    "        return 'Long ago'\n",
    "    else:\n",
    "        return 'Recent'\n",
    "    \n",
    "year_udf = udf(how_long, StringType())\n",
    "df.withColumn('how_long?', year_udf(df['Year'])).show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-------+----+---------+\n",
      "| _c0|Country|Year|      GDP|\n",
      "+----+-------+----+---------+\n",
      "|1813|  Nepal|2015| 743.7653|\n",
      "|1814|  Nepal|2014|  76.2387|\n",
      "|1815|  Nepal|2013|688.61725|\n",
      "|1816|  Nepal|2012| 681.7926|\n",
      "+----+-------+----+---------+\n",
      "only showing top 4 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Writing a fucntion that takes a data frame in input and output \n",
    "\n",
    "def func(df):\n",
    "    return df.filter(df['Country']=='Nepal')\n",
    "\n",
    "func(df).show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-------+----+---------+\n",
      "| _c0|Country|Year|      GDP|\n",
      "+----+-------+----+---------+\n",
      "|1813|  Nepal|2015| 743.7653|\n",
      "|1814|  Nepal|2014|  76.2387|\n",
      "|1815|  Nepal|2013|688.61725|\n",
      "|1816|  Nepal|2012| 681.7926|\n",
      "+----+-------+----+---------+\n",
      "only showing top 4 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Transform transforms the dataframe as indicated by the fucntion \n",
    "# This can be useful is some complex transfomation is needed to \n",
    "# be done in multiple dataframes. So that we don't have to type \n",
    "# that complex transformation everytime. \n",
    "\n",
    "df.transform(func).show(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_c0 \t\t\t 0\n",
      "Country \t\t\t 0\n",
      "Year \t\t\t 0\n",
      "GDP \t\t\t 448\n"
     ]
    }
   ],
   "source": [
    "# Missing values in the dataframe \n",
    "\n",
    "for col_name in df.columns:\n",
    "    print(col_name, '\\t\\t\\t', df.filter(df[col_name].isNull()).count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_c0 \t\t\t 0\n",
      "Country \t\t\t 0\n",
      "Year \t\t\t 0\n",
      "GDP \t\t\t 0\n"
     ]
    }
   ],
   "source": [
    "# Filling the missing values \n",
    "\n",
    "df_nonull = df.fillna({'GDP':7500})\n",
    "\n",
    "for col_name in df_nonull.columns:\n",
    "    print(col_name, '\\t\\t\\t', df_nonull.filter(df_nonull[col_name].isNull()).count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can split the dataset in the given fraction \n",
    "# This can be useful for cross validation \n",
    "\n",
    "train, valid, test = df.randomSplit([3.0, 1.0, 1.0], 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------+----+--------+\n",
      "|_c0|    Country|Year|     GDP|\n",
      "+---+-----------+----+--------+\n",
      "|  2|Afghanistan|2013| 631.745|\n",
      "|  6|Afghanistan|2009|445.8933|\n",
      "|  8|Afghanistan|2007|369.8358|\n",
      "+---+-----------+----+--------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----------+----+--------+\n",
      "|_c0|    Country|Year|     GDP|\n",
      "+---+-----------+----+--------+\n",
      "|  2|Afghanistan|2013| 631.745|\n",
      "|  6|Afghanistan|2009|445.8933|\n",
      "|  8|Afghanistan|2007|369.8358|\n",
      "+---+-----------+----+--------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}