Steam_Data_Processor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Features from the Steam Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Credit: Professor Julian McAuley CSE158 Homework Source Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import ast\n",
    "import os\n",
    "import statistics\n",
    "import sklearn\n",
    "from collections import Counter\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read In Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in the user_items dataset and the steam_games dataset line by line, and store thme into two arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "notebook_path = os.path.abspath(\"Steam_Data_Processor.ipynb\")\n",
    "user_item_file_path = os.path.join(os.path.dirname(notebook_path), \"data/australian_users_items.json\")\n",
    "steam_games_file_path = os.path.join(os.path.dirname(notebook_path), \"data/steam_games.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_items = []\n",
    "with open(user_item_file_path, 'r') as data:\n",
    "    for line in data:\n",
    "        users_items.append(ast.literal_eval(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "games_data = []\n",
    "with open(steam_games_file_path, 'r') as file:\n",
    "    for line in file:\n",
    "        games_data.append(ast.literal_eval(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "games_dict = {}\n",
    "games_no_id = []\n",
    "for game in games_data:\n",
    "    if 'id' in game:\n",
    "        game['popularity'] = 0\n",
    "        game['total_playtime'] = 0\n",
    "        game['owners'] = {}\n",
    "        games_dict[game['id']] = game\n",
    "    else:\n",
    "        games_no_id.append(game)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Extraction for both Users and Items"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract a list of features for users (users_meta_data):\n",
    "- how many games a user owned under each developer\n",
    "- how much time did user spend on each developer's games\n",
    "- what genres does the user prefer (genres - count, playtime)\n",
    "- what tags does the user prefer (tags - count, playtime)\n",
    "- what specs does the user prefer (specs - count, playtime)\n",
    "- total amount of money spent on all games (total_cost)\n",
    "- total amount of time spent on all games (total_playtime)\n",
    "\n",
    "Extract a list of features for items (games_dict):\n",
    "- name (app_name)\n",
    "- developer (developer)\n",
    "- early access (early_access)\n",
    "- genres\n",
    "- id\n",
    "- median playtime (median_playtime)\n",
    "- metascore\n",
    "- owners\n",
    "- popularity\n",
    "- price\n",
    "- publisher\n",
    "- release date (release_date)\n",
    "- reviews url (reviews_url)\n",
    "- sentiment\n",
    "- specs\n",
    "- tags\n",
    "- title\n",
    "- total_playtime\n",
    "- url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractItemPrice(item_id, games_dict):\n",
    "    if 'price' in games_dict[item_id]:\n",
    "        if type(games_dict[item_id]['price']) is float:\n",
    "            return float(games_dict[item_id]['price'])\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collectOwners(item, item_id, user_id, games_dict):\n",
    "    #get dictionary of owners\n",
    "    games_dict[item_id]['owners'][user_id] = item['playtime_forever']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractItemCategoricalData(field, item, item_id, user_id, games_dict, users_meta_data):\n",
    "    # retrieve all the available item data\n",
    "    if field in games_dict[item_id]:\n",
    "        field_set = games_dict[item_id][field]\n",
    "        users_meta_data[user_id][field]['count'] = dict.fromkeys(field_set, 0)\n",
    "        count_meta_data = users_meta_data[user_id][field]['count']\n",
    "        users_meta_data[user_id][field]['playtime'] = dict.fromkeys(field_set, 0)\n",
    "        playtime_meta_data = users_meta_data[user_id][field]['playtime']\n",
    "\n",
    "        for value in field_set:\n",
    "            count_meta_data[value] += 1\n",
    "            playtime_meta_data[value] += item['playtime_forever']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract(item, user_id, games_dict, users_meta_data):\n",
    "    item_id = item['item_id']\n",
    "    users_meta_data[user_id]['total_playtime'] += item['playtime_forever']\n",
    "    \n",
    "    if item_id not in games_dict:\n",
    "        games_dict[item_id] = {\n",
    "            'title': item['item_name'],\n",
    "            'popularity': 0,\n",
    "            'total_playtime': 0,\n",
    "            'owners': {}\n",
    "        }\n",
    "    \n",
    "    collectOwners(item, item_id, user_id, games_dict)\n",
    "    users_meta_data[user_id]['total_cost'] += extractItemPrice(item_id, games_dict)\n",
    "    extractItemCategoricalData('genres', item, item_id, user_id, games_dict, users_meta_data)\n",
    "    extractItemCategoricalData('tags', item, item_id, user_id, games_dict, users_meta_data)\n",
    "    extractItemCategoricalData('specs', item, item_id, user_id, games_dict, users_meta_data)\n",
    "    extractItemCategoricalData('developers', item, item_id, user_id, games_dict, users_meta_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def usersItemsMetaDataExtractor(users_items, games_dict):\n",
    "    users_meta_data = {}\n",
    "    for user in users_items:\n",
    "        user_id = user['user_id']\n",
    "        # create a new user meta_data\n",
    "        users_meta_data[user_id] = {\n",
    "            'genres': {},\n",
    "            'tags': {},\n",
    "            'specs': {},\n",
    "            'developers': {},\n",
    "            'total_playtime': 0,\n",
    "            'total_cost': 0,\n",
    "            'item_count': user['items_count']\n",
    "        }\n",
    "\n",
    "        for item in user['items']:\n",
    "            extract(item, user_id, games_dict, users_meta_data)\n",
    "            \n",
    "    return users_meta_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_meta_data = usersItemsMetaDataExtractor(users_items, games_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def medianItemPlaytime(item):\n",
    "    playtimes = [x for x in list(item['owners'].values()) if x > 0]\n",
    "    if len(playtimes) > 0:\n",
    "        return statistics.median(playtimes)\n",
    "    else:\n",
    "        return 0\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for game in games_dict:\n",
    "    games_dict[game]['median_playtime'] = medianItemPlaytime(games_dict[game])\n",
    "    games_dict[game]['popularity'] = len(games_dict[game]['owners'])\n",
    "    games_dict[game]['total_playtime'] = sum(list(games_dict[game]['owners'].values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_dict = {}\n",
    "for user in users_items:\n",
    "    users_dict[user['user_id']] = user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7664"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "games_dict['70']['popularity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7664"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list(games_dict['70']['owners'].keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2650946"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "games_dict['70']['total_playtime']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2650946"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(list(games_dict['70']['owners'].values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/users_meta_data.json', 'w') as file:\n",
    "    json.dump(users_meta_data, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/items_meta_data.json', 'w') as file:\n",
    "    json.dump(games_dict, file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}