baseline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BASELINE - For this, we are just feeding budget as our prediction, as our initial study indicates that budget has the highest correlation, out of all attributes provided to, revenue."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>belongs_to_collection</th>\n",
       "      <th>budget</th>\n",
       "      <th>genres</th>\n",
       "      <th>homepage</th>\n",
       "      <th>imdb_id</th>\n",
       "      <th>original_language</th>\n",
       "      <th>original_title</th>\n",
       "      <th>overview</th>\n",
       "      <th>popularity</th>\n",
       "      <th>...</th>\n",
       "      <th>production_countries</th>\n",
       "      <th>release_date</th>\n",
       "      <th>runtime</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>status</th>\n",
       "      <th>tagline</th>\n",
       "      <th>title</th>\n",
       "      <th>Keywords</th>\n",
       "      <th>cast</th>\n",
       "      <th>crew</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3001</td>\n",
       "      <td>[{'id': 34055, 'name': 'Pokémon Collection', '...</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': 12, 'name': 'Adventure'}, {'id': 16, '...</td>\n",
       "      <td>http://www.pokemon.com/us/movies/movie-pokemon...</td>\n",
       "      <td>tt1226251</td>\n",
       "      <td>ja</td>\n",
       "      <td>ディアルガVSパルキアVSダークライ</td>\n",
       "      <td>Ash and friends (this time accompanied by newc...</td>\n",
       "      <td>3.851534</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'iso_3166_1': 'JP', 'name': 'Japan'}, {'iso_...</td>\n",
       "      <td>7/14/07</td>\n",
       "      <td>90.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}, {'iso...</td>\n",
       "      <td>Released</td>\n",
       "      <td>Somewhere Between Time &amp; Space... A Legend Is ...</td>\n",
       "      <td>Pokémon: The Rise of Darkrai</td>\n",
       "      <td>[{'id': 11451, 'name': 'pok√©mon'}, {'id': 115...</td>\n",
       "      <td>[{'cast_id': 3, 'character': 'Tonio', 'credit_...</td>\n",
       "      <td>[{'credit_id': '52fe44e7c3a368484e03d683', 'de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3002</td>\n",
       "      <td>NaN</td>\n",
       "      <td>88000</td>\n",
       "      <td>[{'id': 27, 'name': 'Horror'}, {'id': 878, 'na...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tt0051380</td>\n",
       "      <td>en</td>\n",
       "      <td>Attack of the 50 Foot Woman</td>\n",
       "      <td>When an abused wife grows to giant size becaus...</td>\n",
       "      <td>3.559789</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'iso_3166_1': 'US', 'name': 'United States o...</td>\n",
       "      <td>5/19/58</td>\n",
       "      <td>65.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>A titanic beauty spreads a macabre wave of hor...</td>\n",
       "      <td>Attack of the 50 Foot Woman</td>\n",
       "      <td>[{'id': 9748, 'name': 'revenge'}, {'id': 9951,...</td>\n",
       "      <td>[{'cast_id': 2, 'character': 'Nancy Fowler Arc...</td>\n",
       "      <td>[{'credit_id': '55807805c3a3685b1300060b', 'de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3003</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': 35, 'name': 'Comedy'}, {'id': 10749, '...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tt0118556</td>\n",
       "      <td>en</td>\n",
       "      <td>Addicted to Love</td>\n",
       "      <td>Good-natured astronomer Sam is devastated when...</td>\n",
       "      <td>8.085194</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'iso_3166_1': 'US', 'name': 'United States o...</td>\n",
       "      <td>5/23/97</td>\n",
       "      <td>100.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>A Comedy About Lost Loves And Last Laughs</td>\n",
       "      <td>Addicted to Love</td>\n",
       "      <td>[{'id': 931, 'name': 'jealousy'}, {'id': 9673,...</td>\n",
       "      <td>[{'cast_id': 11, 'character': 'Maggie', 'credi...</td>\n",
       "      <td>[{'credit_id': '52fe4330c3a36847f8041367', 'de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3004</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6800000</td>\n",
       "      <td>[{'id': 18, 'name': 'Drama'}, {'id': 10752, 'n...</td>\n",
       "      <td>http://www.sonyclassics.com/incendies/</td>\n",
       "      <td>tt1255953</td>\n",
       "      <td>fr</td>\n",
       "      <td>Incendies</td>\n",
       "      <td>A mother's last wishes send twins Jeanne and S...</td>\n",
       "      <td>8.596012</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'iso_3166_1': 'CA', 'name': 'Canada'}, {'iso...</td>\n",
       "      <td>9/4/10</td>\n",
       "      <td>130.0</td>\n",
       "      <td>[{'iso_639_1': 'fr', 'name': 'Français'}, {'is...</td>\n",
       "      <td>Released</td>\n",
       "      <td>The search began at the opening of their mothe...</td>\n",
       "      <td>Incendies</td>\n",
       "      <td>[{'id': 378, 'name': 'prison'}, {'id': 539, 'n...</td>\n",
       "      <td>[{'cast_id': 6, 'character': 'Nawal', 'credit_...</td>\n",
       "      <td>[{'credit_id': '56478092c3a36826140043af', 'de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3005</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2000000</td>\n",
       "      <td>[{'id': 36, 'name': 'History'}, {'id': 99, 'na...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tt0418753</td>\n",
       "      <td>en</td>\n",
       "      <td>Inside Deep Throat</td>\n",
       "      <td>In 1972, a seemingly typical shoestring budget...</td>\n",
       "      <td>3.217680</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'iso_3166_1': 'US', 'name': 'United States o...</td>\n",
       "      <td>2/11/05</td>\n",
       "      <td>92.0</td>\n",
       "      <td>[{'iso_639_1': 'en', 'name': 'English'}]</td>\n",
       "      <td>Released</td>\n",
       "      <td>It was filmed in 6 days for 25 thousand dollar...</td>\n",
       "      <td>Inside Deep Throat</td>\n",
       "      <td>[{'id': 279, 'name': 'usa'}, {'id': 1228, 'nam...</td>\n",
       "      <td>[{'cast_id': 1, 'character': 'Narrator (voice)...</td>\n",
       "      <td>[{'credit_id': '52fe44ce9251416c75041967', 'de...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     id                              belongs_to_collection   budget  \\\n",
       "0  3001  [{'id': 34055, 'name': 'Pokémon Collection', '...        0   \n",
       "1  3002                                                NaN    88000   \n",
       "2  3003                                                NaN        0   \n",
       "3  3004                                                NaN  6800000   \n",
       "4  3005                                                NaN  2000000   \n",
       "\n",
       "                                              genres  \\\n",
       "0  [{'id': 12, 'name': 'Adventure'}, {'id': 16, '...   \n",
       "1  [{'id': 27, 'name': 'Horror'}, {'id': 878, 'na...   \n",
       "2  [{'id': 35, 'name': 'Comedy'}, {'id': 10749, '...   \n",
       "3  [{'id': 18, 'name': 'Drama'}, {'id': 10752, 'n...   \n",
       "4  [{'id': 36, 'name': 'History'}, {'id': 99, 'na...   \n",
       "\n",
       "                                            homepage    imdb_id  \\\n",
       "0  http://www.pokemon.com/us/movies/movie-pokemon...  tt1226251   \n",
       "1                                                NaN  tt0051380   \n",
       "2                                                NaN  tt0118556   \n",
       "3             http://www.sonyclassics.com/incendies/  tt1255953   \n",
       "4                                                NaN  tt0418753   \n",
       "\n",
       "  original_language               original_title  \\\n",
       "0                ja           ディアルガVSパルキアVSダークライ   \n",
       "1                en  Attack of the 50 Foot Woman   \n",
       "2                en             Addicted to Love   \n",
       "3                fr                    Incendies   \n",
       "4                en           Inside Deep Throat   \n",
       "\n",
       "                                            overview  popularity  ...  \\\n",
       "0  Ash and friends (this time accompanied by newc...    3.851534  ...   \n",
       "1  When an abused wife grows to giant size becaus...    3.559789  ...   \n",
       "2  Good-natured astronomer Sam is devastated when...    8.085194  ...   \n",
       "3  A mother's last wishes send twins Jeanne and S...    8.596012  ...   \n",
       "4  In 1972, a seemingly typical shoestring budget...    3.217680  ...   \n",
       "\n",
       "                                production_countries release_date runtime  \\\n",
       "0  [{'iso_3166_1': 'JP', 'name': 'Japan'}, {'iso_...      7/14/07    90.0   \n",
       "1  [{'iso_3166_1': 'US', 'name': 'United States o...      5/19/58    65.0   \n",
       "2  [{'iso_3166_1': 'US', 'name': 'United States o...      5/23/97   100.0   \n",
       "3  [{'iso_3166_1': 'CA', 'name': 'Canada'}, {'iso...       9/4/10   130.0   \n",
       "4  [{'iso_3166_1': 'US', 'name': 'United States o...      2/11/05    92.0   \n",
       "\n",
       "                                    spoken_languages    status  \\\n",
       "0  [{'iso_639_1': 'en', 'name': 'English'}, {'iso...  Released   \n",
       "1           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "2           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "3  [{'iso_639_1': 'fr', 'name': 'Français'}, {'is...  Released   \n",
       "4           [{'iso_639_1': 'en', 'name': 'English'}]  Released   \n",
       "\n",
       "                                             tagline  \\\n",
       "0  Somewhere Between Time & Space... A Legend Is ...   \n",
       "1  A titanic beauty spreads a macabre wave of hor...   \n",
       "2          A Comedy About Lost Loves And Last Laughs   \n",
       "3  The search began at the opening of their mothe...   \n",
       "4  It was filmed in 6 days for 25 thousand dollar...   \n",
       "\n",
       "                          title  \\\n",
       "0  Pokémon: The Rise of Darkrai   \n",
       "1   Attack of the 50 Foot Woman   \n",
       "2              Addicted to Love   \n",
       "3                     Incendies   \n",
       "4            Inside Deep Throat   \n",
       "\n",
       "                                            Keywords  \\\n",
       "0  [{'id': 11451, 'name': 'pok√©mon'}, {'id': 115...   \n",
       "1  [{'id': 9748, 'name': 'revenge'}, {'id': 9951,...   \n",
       "2  [{'id': 931, 'name': 'jealousy'}, {'id': 9673,...   \n",
       "3  [{'id': 378, 'name': 'prison'}, {'id': 539, 'n...   \n",
       "4  [{'id': 279, 'name': 'usa'}, {'id': 1228, 'nam...   \n",
       "\n",
       "                                                cast  \\\n",
       "0  [{'cast_id': 3, 'character': 'Tonio', 'credit_...   \n",
       "1  [{'cast_id': 2, 'character': 'Nancy Fowler Arc...   \n",
       "2  [{'cast_id': 11, 'character': 'Maggie', 'credi...   \n",
       "3  [{'cast_id': 6, 'character': 'Nawal', 'credit_...   \n",
       "4  [{'cast_id': 1, 'character': 'Narrator (voice)...   \n",
       "\n",
       "                                                crew  \n",
       "0  [{'credit_id': '52fe44e7c3a368484e03d683', 'de...  \n",
       "1  [{'credit_id': '55807805c3a3685b1300060b', 'de...  \n",
       "2  [{'credit_id': '52fe4330c3a36847f8041367', 'de...  \n",
       "3  [{'credit_id': '56478092c3a36826140043af', 'de...  \n",
       "4  [{'credit_id': '52fe44ce9251416c75041967', 'de...  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_data():\n",
    "    #Get the training and testing data\n",
    "    return [pd.read_csv('./data/train.csv'), pd.read_csv('./data/test.csv')]\n",
    "\n",
    "def drop(to_drop, test):\n",
    "    #Drop specified attributes from the training and test data sets\n",
    "    for x in to_drop:\n",
    "        try:\n",
    "            if x != 'revenue':\n",
    "                test = test.drop(x, axis = 1)\n",
    "        except:\n",
    "            pass\n",
    "    return test\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "_, test_df = get_data()\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4398 entries, 0 to 4397\n",
      "Data columns (total 22 columns):\n",
      "id                       4398 non-null int64\n",
      "belongs_to_collection    877 non-null object\n",
      "budget                   4398 non-null int64\n",
      "genres                   4382 non-null object\n",
      "homepage                 1420 non-null object\n",
      "imdb_id                  4398 non-null object\n",
      "original_language        4398 non-null object\n",
      "original_title           4398 non-null object\n",
      "overview                 4384 non-null object\n",
      "popularity               4398 non-null float64\n",
      "poster_path              4397 non-null object\n",
      "production_companies     4140 non-null object\n",
      "production_countries     4296 non-null object\n",
      "release_date             4397 non-null object\n",
      "runtime                  4394 non-null float64\n",
      "spoken_languages         4356 non-null object\n",
      "status                   4396 non-null object\n",
      "tagline                  3535 non-null object\n",
      "title                    4395 non-null object\n",
      "Keywords                 4005 non-null object\n",
      "cast                     4385 non-null object\n",
      "crew                     4376 non-null object\n",
      "dtypes: float64(2), int64(2), object(18)\n",
      "memory usage: 756.0+ KB\n"
     ]
    }
   ],
   "source": [
    "test_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ALL ATTRIBUTES\n",
    "ALL_ATTRIBUTES = ['id','belongs_to_collection','budget','genres','homepage','imdb_id','original_language',\n",
    "                 'original_title','overview','popularity','poster_path','production_companies',\n",
    "                 'production_countries','release_date','runtime','spoken_languages', 'status', 'tagline',\n",
    "                 'title','Keywords','cast','crew','revenue']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_drop = ['id','belongs_to_collection','genres','homepage','imdb_id','original_language',\n",
    "                 'original_title','overview','popularity','poster_path','production_companies',\n",
    "                 'production_countries','release_date','runtime','spoken_languages', 'status', 'tagline',\n",
    "                 'title','Keywords','cast','crew']\n",
    "test_df = drop(to_drop, test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>budget</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>88000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    budget\n",
       "0        0\n",
       "1    88000\n",
       "2        0\n",
       "3  6800000\n",
       "4  2000000"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>budget</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>88000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    budget\n",
       "0        0\n",
       "1    88000\n",
       "2        0\n",
       "3  6800000\n",
       "4  2000000"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "def f(x):\n",
    "    if x<0:\n",
    "        return 0\n",
    "    return x\n",
    "\n",
    "#test_df = test_df.apply(np.log)\n",
    "test_df = test_df.applymap(f)\n",
    "test_df = test_df.fillna(0)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_df = pd.read_csv('./data/test.csv')\n",
    "for x in ALL_ATTRIBUTES:\n",
    "    try:\n",
    "        if x != 'revenue'and x != 'id':\n",
    "            output_df = output_df.drop(x, axis=1)\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "output_df['revenue'] = test_df\n",
    "output_df.to_csv(\"./submission.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On Kaggle, this produced a MSLE of "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}