ETL_EDA_ML.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import the libraries for ETL, EDA and ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I define a function to set the drive url to download csv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def url_set(url: str):\n",
    "    return 'https://drive.google.com/uc?id=' + url.split('/')[-2]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I create a function to generate the index id in the same where I create the dataframe of the platforms information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_id(show_id, platform):\n",
    "    return platform[0] + show_id\n",
    "\n",
    "dic_url = {\"amazon_prime\": url_set(\"https://drive.google.com/file/d/1LJIYUiPnFbU0mOKQMW54QZPPq1bAd5Zc/view?usp=share_link\"),\n",
    "            \"disney_plus\": url_set(\"https://drive.google.com/file/d/1d8BTVBj3NmCxUMknTkVcuVRWwVhXthiP/view?usp=share_link\"),\n",
    "           \"hulu\": url_set(\"https://drive.google.com/file/d/1Sy7HMCQgVlT31CAD2ewXrN82jMlsK21s/view?usp=share_link\"),\n",
    "           \"netflix\": url_set(\"https://drive.google.com/file/d/1yQ44qjfACWsR66lb-D_tDCmmDxPnIXvm/view?usp=share_link\")}\n",
    "\n",
    "df_names = pd.DataFrame()\n",
    "\n",
    "for key, value in dic_url.items():\n",
    "    # Get CSV file from Google Drive without downloading it\n",
    "    df = pd.read_csv(value)\n",
    "    df[\"id\"] = df[\"show_id\"].apply(lambda x: generate_id(x, key))\n",
    "    df_names = pd.concat([df_names, df])\n",
    "\n",
    "df_names.reset_index(drop=True, inplace=True)\n",
    "\n",
    "list_url = [url_set(\"https://drive.google.com/file/d/1ImYbz29myZKGDZCYY4r5yzXWcdPqp-DL/view?usp=share_link\"),\n",
    "        url_set(\"https://drive.google.com/file/d/1rlpHWaxvo5kX5hyyOP7i5zpC5V9RfsaV/view?usp=share_link\"),\n",
    "        url_set(\"https://drive.google.com/file/d/1CBZA4xkDhfa-CX8dr92rhjKR3HhgctzE/view?usp=share_link\"),\n",
    "        url_set(\"https://drive.google.com/file/d/1CsaTyLVB-AZ78yJp9XeSw2qvTSNj9aYx/view?usp=share_link\"),\n",
    "        url_set(\"https://drive.google.com/file/d/1QztUrbE6CEC57AgbNcR9XJf9WIvwUF3P/view?usp=share_link\"),\n",
    "        url_set(\"https://drive.google.com/file/d/1y0TKNdKhSumjjaDSUGQTPIlMZuFqy1Zn/view?usp=share_link\"),\n",
    "        url_set(\"https://drive.google.com/file/d/18WgvpsLVK_5uhCJm5HyZSytNreOIRiT1/view?usp=share_link\"),\n",
    "        url_set(\"https://drive.google.com/file/d/1dwqAfTL7BXbOvJn_A3bwQkL9_gbIFoTz/view?usp=share_link\")]\n",
    "\n",
    "#Merging multiple csv files into a single dataframe\n",
    "df_ratings = pd.concat(map(pd.read_csv, list_url))\n",
    "\n",
    "df_ratings.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I replace the null values of the rating field with the string \"G\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names[\"rating\"] = df_names[\"rating\"].fillna(\"G\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Change the date format to YYYYY-mm-dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names[\"date_added\"] = df_names[\"date_added\"].str.strip()\n",
    "df_names[\"date_added\"] = pd.to_datetime(df_names[\"date_added\"], format=\"%B %d, %Y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings[\"timestamp\"] = pd.to_datetime(df_ratings[\"timestamp\"], unit=\"s\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The text fields are converted to lowercase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = [\"type\", \"title\", \"director\", \"cast\", \"country\", \"listed_in\", \"description\", \"rating\"]\n",
    "df_names[cols] = df_names[cols].apply(lambda x: x.str.lower())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split duration into two separate columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_duration = df_names[\"duration\"].str.split(\" \", n=1, expand=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assign the resulting columns to duration_int and duration_type respectively"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names[\"duration_int\"] = df_duration[0].apply(lambda x: x if pd.notnull(x) else None)\n",
    "df_names[\"duration_type\"] = df_duration[1].apply(lambda x: x if pd.notnull(x) else None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['g', '13+', 'all', '18+', 'r', 'tv-y', 'tv-y7', 'nr', '16+',\n",
       "       'tv-pg', '7+', 'tv-14', 'tv-nr', 'tv-g', 'pg-13', 'tv-ma', 'pg',\n",
       "       'nc-17', 'unrated', '16', 'ages_16_', 'ages_18_', 'all_ages',\n",
       "       'not_rate', 'tv-y7-fv', 'not rated', '2 seasons', '93 min',\n",
       "       '4 seasons', '136 min', '91 min', '85 min', '98 min', '89 min',\n",
       "       '94 min', '86 min', '3 seasons', '121 min', '88 min', '101 min',\n",
       "       '1 season', '83 min', '100 min', '95 min', '92 min', '96 min',\n",
       "       '109 min', '99 min', '75 min', '87 min', '67 min', '104 min',\n",
       "       '107 min', '84 min', '103 min', '105 min', '119 min', '114 min',\n",
       "       '82 min', '90 min', '130 min', '110 min', '80 min', '6 seasons',\n",
       "       '97 min', '111 min', '81 min', '49 min', '45 min', '41 min',\n",
       "       '73 min', '40 min', '36 min', '39 min', '34 min', '47 min',\n",
       "       '65 min', '37 min', '78 min', '102 min', '129 min', '115 min',\n",
       "       '112 min', '61 min', '106 min', '76 min', '77 min', '79 min',\n",
       "       '157 min', '28 min', '64 min', '7 min', '5 min', '6 min',\n",
       "       '127 min', '142 min', '108 min', '57 min', '118 min', '116 min',\n",
       "       '12 seasons', '71 min', '74 min', '66 min', 'ur'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names['rating'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalize rating column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names[\"rating\"] = df_names[\"rating\"].astype(str)\n",
    "for rating in df_names['rating'].unique():\n",
    "    if rating.endswith('min'):\n",
    "        duration_int, duration_type = rating.split(\" \")\n",
    "        filter = df_names['rating'] == rating\n",
    "        df_names.loc[filter, 'duration_int'] = int(duration_int)\n",
    "        df_names.loc[filter, 'duration_type'] = duration_type\n",
    "        df_names.loc[filter, 'rating'] = 'unrated'\n",
    "    elif rating.endswith('season') or rating.endswith('seasons'):\n",
    "        filter = df_names['rating'] == rating\n",
    "        df_names.loc[filter, 'rating'] = 'unrated'\n",
    "\n",
    "df_names['rating'] = df_names['rating'].replace('all', 'all_ages')\n",
    "df_names['rating'] = df_names['rating'].replace('nr', 'unrated')\n",
    "df_names['rating'] = df_names['rating'].replace('tv-pg', 'pg')\n",
    "df_names['rating'] = df_names['rating'].replace('tv-nr', 'unrated')\n",
    "df_names['rating'] = df_names['rating'].replace('tv-g', 'g')\n",
    "df_names['rating'] = df_names['rating'].replace('tv-ma', '18+')\n",
    "df_names['rating'] = df_names['rating'].replace('nc-17', '18+')\n",
    "df_names['rating'] = df_names['rating'].replace('16', '16+')\n",
    "df_names['rating'] = df_names['rating'].replace('ages_16_', '16+')\n",
    "df_names['rating'] = df_names['rating'].replace('18', '18+')\n",
    "df_names['rating'] = df_names['rating'].replace('ages_18_', '18+')\n",
    "df_names['rating'] = df_names['rating'].replace('not_rate', 'unrated')\n",
    "df_names['rating'] = df_names['rating'].replace('not rated', 'unrated')\n",
    "df_names['rating'] = df_names['rating'].replace('ur', 'unrated')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I eliminate the column \"show_id\", it does not provide more information than \"id\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names.drop(columns=\"show_id\", inplace=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I eliminate the \"duration\" column that separates your information in two."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names.drop(columns=\"duration\", inplace=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I eliminate the \"description\" column, which is difficult to process and convert into a form that is suitable for the Machine Learning model.\n",
    "Text processing may require more advanced techniques that are not feasible for the project in question.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names.drop(columns=\"description\", inplace=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I remove the column \"date_added\", since the column \"release_year\" describes the year, then the date when the entry was added to the database does not provide additional information to the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names.drop(columns=\"date_added\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22998, 11)\n",
      "(22179, 11)\n"
     ]
    }
   ],
   "source": [
    "print(df_names.shape)\n",
    "df_names.drop_duplicates(subset=[\"type\", \"title\"] ,inplace=True)\n",
    "print(df_names.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11024289, 4)\n",
      "(11024165, 4)\n"
     ]
    }
   ],
   "source": [
    "print(df_ratings.shape)\n",
    "df_ratings.drop_duplicates(inplace=True)\n",
    "print(df_ratings.shape)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I remove the column \"timestamp\", since the column \"release_year\" describes the year, then the date when the entry was added to the database does not provide additional information to the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings.drop(columns='timestamp', inplace=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Group by movieId and average the scores of different users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings_grouped = round(df_ratings.groupby('movieId', as_index=False)['rating'].mean(),1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I change the name of the column to perform the merge."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ratings_grouped = df_ratings_grouped.rename(columns={'rating': 'scored'})"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge with the general platform dataframe from the movieId."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names = pd.merge(df_names, df_ratings_grouped[['movieId', 'scored']], left_on='id', right_on='movieId')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I discard the \"movieId\" column that I used to perform the merge."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names.drop(columns=\"movieId\", inplace=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I use the .info() method to know which columns have null values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 22179 entries, 0 to 22178\n",
      "Data columns (total 12 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   type           22179 non-null  object \n",
      " 1   title          22179 non-null  object \n",
      " 2   director       14315 non-null  object \n",
      " 3   cast           17109 non-null  object \n",
      " 4   country        10795 non-null  object \n",
      " 5   release_year   22179 non-null  int64  \n",
      " 6   rating         22179 non-null  object \n",
      " 7   listed_in      22179 non-null  object \n",
      " 8   id             22179 non-null  object \n",
      " 9   duration_int   21940 non-null  object \n",
      " 10  duration_type  21940 non-null  object \n",
      " 11  scored         22179 non-null  float64\n",
      "dtypes: float64(1), int64(1), object(10)\n",
      "memory usage: 2.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df_names.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I modify the formats of some columns of the platforms dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names[\"id\"] = df_names[\"id\"].astype(str)\n",
    "df_names[\"type\"] = df_names[\"type\"].fillna(\"\").astype(str)\n",
    "df_names[\"country\"] = df_names[\"country\"].fillna(\"\").astype(str)\n",
    "df_names[\"director\"] = df_names[\"director\"].fillna(\"\").astype(str)\n",
    "df_names[\"cast\"] = df_names[\"cast\"].fillna(\"\").astype(str)\n",
    "df_names[\"rating\"] = df_names[\"rating\"].fillna(\"\").astype(str)\n",
    "df_names[\"duration_type\"] = df_names[\"duration_type\"].fillna(\"\").astype(str)\n",
    "df_names[\"release_year\"] = df_names[\"release_year\"].fillna(0).astype(int)\n",
    "df_names[\"duration_int\"] = df_names[\"duration_int\"].fillna(0).astype(int)\n",
    "df_names[\"scored\"] = df_names[\"scored\"].astype(float)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The ratings dataset has 115077 unique users\n"
     ]
    }
   ],
   "source": [
    "print('The ratings dataset has', df_ratings['userId'].nunique(), 'unique users')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of movies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The ratings dataset has 22998 unique movies\n"
     ]
    }
   ],
   "source": [
    "print('The ratings dataset has', df_ratings['movieId'].nunique(), 'unique movies')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of ratings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The ratings dataset has 10 unique ratings\n"
     ]
    }
   ],
   "source": [
    "print('The ratings dataset has', df_ratings['rating'].nunique(), 'unique ratings')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "List of unique ratings.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The unique ratings are [0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0]\n"
     ]
    }
   ],
   "source": [
    "print('The unique ratings are', sorted(df_ratings['rating'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['comedy, drama', 'drama, international', 'action, drama, suspense',\n",
       "       ..., 'comedies, independent movies, thrillers',\n",
       "       'international movies, lgbtq movies, romantic movies',\n",
       "       'cult movies, dramas, thrillers'], dtype=object)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names.listed_in.unique()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I remove the columns I do not use in the functions and create a dataset for the ML."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names.drop(columns=[\"director\"], inplace=True)\n",
    "df_ml = df_names.copy()\n",
    "df_ml = df_ml[df_ml[\"type\"] == \"movie\"]\n",
    "df_ml.drop(columns=[\"cast\", \"country\", \"release_year\", \"rating\", \"duration_int\", \"rating\", \"duration_type\", \"scored\", \"type\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.drop(columns=\"id\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['comedy, drama', 'drama, international', 'action, drama, suspense',\n",
       "       ..., 'comedies, independent movies, thrillers',\n",
       "       'international movies, lgbtq movies, romantic movies',\n",
       "       'cult movies, dramas, thrillers'], dtype=object)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ml.listed_in.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "drama                             908\n",
       "comedy                            487\n",
       "documentaries                     483\n",
       "comedy, drama                     370\n",
       "drama, suspense                   349\n",
       "                                 ... \n",
       "animation, anthology, fantasy       1\n",
       "buddy, coming of age, drama         1\n",
       "drama, family, mystery              1\n",
       "animation, comedy, disaster         1\n",
       "cult movies, dramas, thrillers      1\n",
       "Name: listed_in, Length: 1096, dtype: int64"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ml.listed_in.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('dramas'), 'listed_in'] = 'drama'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('lifestyle & culture'), 'listed_in'] = 'sports'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('suspense'), 'listed_in'] = 'suspense'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('comedy'), 'listed_in'] = 'comedy'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('health & wellness'), 'listed_in'] = 'sports'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('kids'), 'listed_in'] = 'kids'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('science fiction'), 'listed_in'] = 'science fiction'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('horror'), 'listed_in'] = 'horror'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('western'), 'listed_in'] = 'western'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.loc[df_ml['listed_in'].str.contains('documentary'), 'listed_in'] = 'documentary'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "504"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ml.listed_in.nunique()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Maching Learning Model from TfidVectorizer and cosine_similarity is based on the construction of a feature matrix from the text documents of a dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recommended movies based on tarzan:\n",
      "1. on happiness road\n",
      "2. invincible\n",
      "3. metal skin panic madox-1\n",
      "4. series before 1c onboarding - 2\n",
      "5. nuttiest nutcracker\n"
     ]
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "#Construct the required TF-IDF matrix by fitting and transforming the data\n",
    "vectorizer_matrix = vectorizer.fit_transform(df_names['listed_in'])\n",
    "\n",
    "#Compute the cosine similarity matrix\n",
    "cosine_sim = cosine_similarity(vectorizer_matrix)\n",
    "\n",
    "#Define a function to get movie recommendations based on the cosine similarity score\n",
    "def get_recommendations(title, cosine_sim=cosine_sim, df_names=df_names, top=5):\n",
    "    #Get the index of the movie that matches the title\n",
    "    idx = df_names[df_names['title'] == title].index[0]\n",
    "\n",
    "    #Get the cosine similarity scores of all movies with the given movie\n",
    "    sim_scores = list(enumerate(cosine_sim[idx]))\n",
    "\n",
    "    #Sort the movies based on the cosine similarity scores\n",
    "    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)\n",
    "    \n",
    "    #Loop until we have 5 recommended movies (excluding itself)\n",
    "    movie_indices = []\n",
    "    count = 0\n",
    "    while len(movie_indices) < top:\n",
    "        idx = sim_scores[count][0]\n",
    "        if df_names.iloc[idx]['title'] != title: #Exclude the same movie\n",
    "            movie_indices.append(idx)\n",
    "        count += 1\n",
    "    \n",
    "    #Return the top n most similar movies\n",
    "    return df_names.iloc[movie_indices]['title'].values\n",
    "\n",
    "# get movie recommendations based on user input\n",
    "title = \"tarzan\"\n",
    "title = title.lower()\n",
    "recommendations = get_recommendations(title)\n",
    "\n",
    "# print the recommendations\n",
    "print(f\"Recommended movies based on {title}:\")\n",
    "for i, movie in enumerate(recommendations):\n",
    "    print(f\"{i+1}. {movie}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>country</th>\n",
       "      <th>release_year</th>\n",
       "      <th>rating</th>\n",
       "      <th>listed_in</th>\n",
       "      <th>id</th>\n",
       "      <th>duration_int</th>\n",
       "      <th>duration_type</th>\n",
       "      <th>scored</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>movie</td>\n",
       "      <td>the grand seduction</td>\n",
       "      <td>brendan gleeson, taylor kitsch, gordon pinsent</td>\n",
       "      <td>canada</td>\n",
       "      <td>2014</td>\n",
       "      <td>g</td>\n",
       "      <td>comedy, drama</td>\n",
       "      <td>as1</td>\n",
       "      <td>113</td>\n",
       "      <td>min</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>movie</td>\n",
       "      <td>take care good night</td>\n",
       "      <td>mahesh manjrekar, abhay mahajan, sachin khedekar</td>\n",
       "      <td>india</td>\n",
       "      <td>2018</td>\n",
       "      <td>13+</td>\n",
       "      <td>drama, international</td>\n",
       "      <td>as2</td>\n",
       "      <td>110</td>\n",
       "      <td>min</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>movie</td>\n",
       "      <td>secrets of deception</td>\n",
       "      <td>tom sizemore, lorenzo lamas, robert lasardo, r...</td>\n",
       "      <td>united states</td>\n",
       "      <td>2017</td>\n",
       "      <td>g</td>\n",
       "      <td>action, drama, suspense</td>\n",
       "      <td>as3</td>\n",
       "      <td>74</td>\n",
       "      <td>min</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>movie</td>\n",
       "      <td>pink: staying true</td>\n",
       "      <td>interviews with: pink, adele, beyoncé, britney...</td>\n",
       "      <td>united states</td>\n",
       "      <td>2014</td>\n",
       "      <td>g</td>\n",
       "      <td>documentary</td>\n",
       "      <td>as4</td>\n",
       "      <td>69</td>\n",
       "      <td>min</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>movie</td>\n",
       "      <td>monster maker</td>\n",
       "      <td>harry dean stanton, kieran o'brien, george cos...</td>\n",
       "      <td>united kingdom</td>\n",
       "      <td>1989</td>\n",
       "      <td>g</td>\n",
       "      <td>drama, fantasy</td>\n",
       "      <td>as5</td>\n",
       "      <td>45</td>\n",
       "      <td>min</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    type                 title  \\\n",
       "0  movie   the grand seduction   \n",
       "1  movie  take care good night   \n",
       "2  movie  secrets of deception   \n",
       "3  movie    pink: staying true   \n",
       "4  movie         monster maker   \n",
       "\n",
       "                                                cast         country  \\\n",
       "0     brendan gleeson, taylor kitsch, gordon pinsent          canada   \n",
       "1   mahesh manjrekar, abhay mahajan, sachin khedekar           india   \n",
       "2  tom sizemore, lorenzo lamas, robert lasardo, r...   united states   \n",
       "3  interviews with: pink, adele, beyoncé, britney...   united states   \n",
       "4  harry dean stanton, kieran o'brien, george cos...  united kingdom   \n",
       "\n",
       "   release_year rating                listed_in   id  duration_int  \\\n",
       "0          2014      g            comedy, drama  as1           113   \n",
       "1          2018    13+     drama, international  as2           110   \n",
       "2          2017      g  action, drama, suspense  as3            74   \n",
       "3          2014      g              documentary  as4            69   \n",
       "4          1989      g           drama, fantasy  as5            45   \n",
       "\n",
       "  duration_type  scored  \n",
       "0           min     3.5  \n",
       "1           min     3.5  \n",
       "2           min     3.5  \n",
       "3           min     3.5  \n",
       "4           min     3.5  "
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>listed_in</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the grand seduction</td>\n",
       "      <td>comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>take care good night</td>\n",
       "      <td>drama, international</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>secrets of deception</td>\n",
       "      <td>suspense</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pink: staying true</td>\n",
       "      <td>documentary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>monster maker</td>\n",
       "      <td>drama, fantasy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  title             listed_in\n",
       "0   the grand seduction                comedy\n",
       "1  take care good night  drama, international\n",
       "2  secrets of deception              suspense\n",
       "3    pink: staying true           documentary\n",
       "4         monster maker        drama, fantasy"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ml.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15944, 2)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ml.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>listed_in</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>david's mother</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>zandalee</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>young lovers</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>you're not you</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>words on bathroom walls</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22170</th>\n",
       "      <td>zed plus</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22171</th>\n",
       "      <td>zenda</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22173</th>\n",
       "      <td>zinzana</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22174</th>\n",
       "      <td>zodiac</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22178</th>\n",
       "      <td>zubaan</td>\n",
       "      <td>drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3160 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         title listed_in\n",
       "9               david's mother     drama\n",
       "24                    zandalee     drama\n",
       "32                young lovers     drama\n",
       "36              you're not you     drama\n",
       "73     words on bathroom walls     drama\n",
       "...                        ...       ...\n",
       "22170                 zed plus     drama\n",
       "22171                    zenda     drama\n",
       "22173                  zinzana     drama\n",
       "22174                   zodiac     drama\n",
       "22178                   zubaan     drama\n",
       "\n",
       "[3160 rows x 2 columns]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ml[ df_ml[\"listed_in\"] == \"drama\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ml.to_csv(\"df_ml.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names.to_csv(\"df_platform.csv\", index=False)\n",
    "df_ratings.to_csv(\"df_rating.csv\", index=False)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}