2-9 텍스트분석3_캐글실습.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mercari Pricee SuggestionChallenge\n",
    "유명한 캐글 Challenge중 하나. 일본의 대형 온라인 쇼핑몰인 Mercari사의 제품에 대해 가격을 예측하는 과제\n",
    "\n",
    "- https://www.kaggle.com/c/mercari-price-suggestion-challenge/overview\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_id</th>\n",
       "      <th>name</th>\n",
       "      <th>item_condition_id</th>\n",
       "      <th>category_name</th>\n",
       "      <th>brand_name</th>\n",
       "      <th>price</th>\n",
       "      <th>shipping</th>\n",
       "      <th>item_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>MLB Cincinnati Reds T Shirt Size XL</td>\n",
       "      <td>3</td>\n",
       "      <td>Men/Tops/T-shirts</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>No description yet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Razer BlackWidow Chroma Keyboard</td>\n",
       "      <td>3</td>\n",
       "      <td>Electronics/Computers &amp; Tablets/Components &amp; P...</td>\n",
       "      <td>Razer</td>\n",
       "      <td>52.0</td>\n",
       "      <td>0</td>\n",
       "      <td>This keyboard is in great condition and works ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>AVA-VIV Blouse</td>\n",
       "      <td>1</td>\n",
       "      <td>Women/Tops &amp; Blouses/Blouse</td>\n",
       "      <td>Target</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>Adorable top with a hint of lace and a key hol...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train_id                                 name  item_condition_id  \\\n",
       "0         0  MLB Cincinnati Reds T Shirt Size XL                  3   \n",
       "1         1     Razer BlackWidow Chroma Keyboard                  3   \n",
       "2         2                       AVA-VIV Blouse                  1   \n",
       "\n",
       "                                       category_name brand_name  price  \\\n",
       "0                                  Men/Tops/T-shirts        NaN   10.0   \n",
       "1  Electronics/Computers & Tablets/Components & P...      Razer   52.0   \n",
       "2                        Women/Tops & Blouses/Blouse     Target   10.0   \n",
       "\n",
       "   shipping                                   item_description  \n",
       "0         1                                 No description yet  \n",
       "1         0  This keyboard is in great condition and works ...  \n",
       "2         1  Adorable top with a hint of lace and a key hol...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge, LogisticRegression\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "import pandas as pd\n",
    "\n",
    "mercari_df=pd.read_csv('mercari_train.tsv',sep='\\t')\n",
    "mercari_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1482535 entries, 0 to 1482534\n",
      "Data columns (total 8 columns):\n",
      "train_id             1482535 non-null int64\n",
      "name                 1482535 non-null object\n",
      "item_condition_id    1482535 non-null int64\n",
      "category_name        1476208 non-null object\n",
      "brand_name           849853 non-null object\n",
      "price                1482535 non-null float64\n",
      "shipping             1482535 non-null int64\n",
      "item_description     1482531 non-null object\n",
      "dtypes: float64(1), int64(3), object(4)\n",
      "memory usage: 90.5+ MB\n"
     ]
    }
   ],
   "source": [
    "mercari_df.info()\n",
    "# category_name, brand_name, item_description  null값 존재"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x223332f4898>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEKCAYAAADXdbjqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFwVJREFUeJzt3X+QZWWd3/H3JzML/liRAUaLnYHMGGdNRmo3Ygtj3LW2xIKBGIckmoBWmHKpTNbgRmNScYhVy5bmD80my4ZE2RCZMGxUIKwWU1viOIVWLBMZaRD5NeC06MIsLIwOIlk3urjf/HGfdi/D7W66n+6+M/J+Vd26537Pc87z9Onb/enz455OVSFJUo+/Nu4BSJKOfoaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuK8c9gOVy0kkn1bp168Y9DEk6qtx+++3frarVc7V73oTJunXrmJycHPcwJOmokuSPn0s7D3NJkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuj1vPgHf41N7H5px3jvOPHUZRyJJRyb3TCRJ3QwTSVI3w0SS1M0wkSR1M0wkSd0ME0lStznDJMmOJI8nuWeo9jtJ7k9yV5LPJjl+aN6lSaaSPJDknKH65labSrJ9qL4+yd4k+5Ncn+SYVj+2vZ5q89fN1YckaTyey57JNcDmw2p7gNOq6peAbwKXAiTZCFwAvLot8/EkK5KsAD4GnAtsBC5sbQE+ClxeVRuAJ4CLW/1i4ImqeiVweWs3Yx/z/LolSYtozjCpqi8Dhw6rfaGqnm4vbwXWtuktwHVV9aOq+jYwBZzRHlNV9WBV/Ri4DtiSJMCbgBvb8juB84fWtbNN3wic1drP1IckaUwW45zJrwM3t+k1wMND8w602kz1E4HvDwXTdP0Z62rzn2ztZ1rXsyTZlmQyyeTBgwcX9MVJkubWFSZJPgg8DXxyujSiWS2gvpB1PbtYdVVVTVTVxOrVq0c1kSQtggXfmyvJVuAtwFlVNf3L/ABwylCztcAjbXpU/bvA8UlWtr2P4fbT6zqQZCXwUgaH22brQ5I0BgvaM0myGfgA8Naq+uHQrF3ABe1KrPXABuBrwG3Ahnbl1jEMTqDvaiH0JeBtbfmtwE1D69rapt8GfLG1n6kPSdKYzLlnkuTTwK8BJyU5AFzG4OqtY4E9g3Pi3FpVv1FV9ya5AbiPweGvS6rqJ2097wF2AyuAHVV1b+viA8B1Sf4d8HXg6la/GviDJFMM9kguAJitD0nSeOSvjlD9bJuYmKjJyckFLest6CU9XyW5vaom5mrnJ+AlSd0ME0lSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3QwTSVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3QwTSVI3w0SS1G3OMEmyI8njSe4Zqp2QZE+S/e15VasnyRVJppLcleT0oWW2tvb7k2wdqr82yd1tmSuSZKF9SJLG47nsmVwDbD6sth24pao2ALe01wDnAhvaYxtwJQyCAbgMOBM4A7hsOhxam21Dy21eSB+SpPGZM0yq6svAocPKW4CdbXoncP5Q/doauBU4PsnJwDnAnqo6VFVPAHuAzW3ecVX11aoq4NrD1jWfPiRJY7LQcyYvr6pHAdrzy1p9DfDwULsDrTZb/cCI+kL6kCSNyWKfgM+IWi2gvpA+nt0w2ZZkMsnkwYMH51itJGmhFhomj00fWmrPj7f6AeCUoXZrgUfmqK8dUV9IH89SVVdV1URVTaxevXpeX6Ak6blbaJjsAqavyNoK3DRUv6hdcbUJeLIdotoNnJ1kVTvxfjawu817KsmmdhXXRYetaz59SJLGZOVcDZJ8Gvg14KQkBxhclfUR4IYkFwMPAW9vzT8HnAdMAT8E3gVQVYeSfBi4rbX7UFVNn9R/N4Mrxl4I3NwezLcPSdL4zBkmVXXhDLPOGtG2gEtmWM8OYMeI+iRw2oj69+bbhyRpPPwEvCSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG5dYZLkXya5N8k9ST6d5AVJ1ifZm2R/kuuTHNPaHtteT7X564bWc2mrP5DknKH65labSrJ9qD6yD0nSeCw4TJKsAf4FMFFVpwErgAuAjwKXV9UG4Ang4rbIxcATVfVK4PLWjiQb23KvBjYDH0+yIskK4GPAucBG4MLWlln6kCSNQe9hrpXAC5OsBF4EPAq8Cbixzd8JnN+mt7TXtPlnJUmrX1dVP6qqbwNTwBntMVVVD1bVj4HrgC1tmZn6kCSNwYLDpKr+BPgPwEMMQuRJ4Hbg+1X1dGt2AFjTptcAD7dln27tTxyuH7bMTPUTZ+lDkjQGPYe5VjHYq1gP/ALwYgaHpA5X04vMMG+x6qPGuC3JZJLJgwcPjmoiSVoEPYe53gx8u6oOVtVfAJ8B/g5wfDvsBbAWeKRNHwBOAWjzXwocGq4ftsxM9e/O0sczVNVVVTVRVROrV6/u+FIlSbPpCZOHgE1JXtTOY5wF3Ad8CXhba7MVuKlN72qvafO/WFXV6he0q73WAxuArwG3ARvalVvHMDhJv6stM1MfkqQx6DlnspfBSfA7gLvbuq4CPgC8P8kUg/MbV7dFrgZObPX3A9vbeu4FbmAQRJ8HLqmqn7RzIu8BdgP7gBtaW2bpQ5I0Bhn8of+zb2JioiYnJxe07Kf2PjTjvHeceepChyRJR7wkt1fVxFzt/AS8JKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkbl1hkuT4JDcmuT/JviSvT3JCkj1J9rfnVa1tklyRZCrJXUlOH1rP1tZ+f5KtQ/XXJrm7LXNFkrT6yD4kSePRu2fyn4DPV9XfBH4Z2AdsB26pqg3ALe01wLnAhvbYBlwJg2AALgPOBM4ALhsKhytb2+nlNrf6TH1IksZgwWGS5DjgjcDVAFX146r6PrAF2Nma7QTOb9NbgGtr4Fbg+CQnA+cAe6rqUFU9AewBNrd5x1XVV6uqgGsPW9eoPiRJY9CzZ/IK4CDw35N8PcknkrwYeHlVPQrQnl/W2q8BHh5a/kCrzVY/MKLOLH1IksagJ0xWAqcDV1bVa4A/Y/bDTRlRqwXUn7Mk25JMJpk8ePDgfBaVJM1DT5gcAA5U1d72+kYG4fJYO0RFe358qP0pQ8uvBR6Zo752RJ1Z+niGqrqqqiaqamL16tUL+iIlSXNbcJhU1Z8CDyd5VSudBdwH7AKmr8jaCtzUpncBF7WrujYBT7ZDVLuBs5OsaifezwZ2t3lPJdnUruK66LB1jepDkjQGKzuX/03gk0mOAR4E3sUgoG5IcjHwEPD21vZzwHnAFPDD1paqOpTkw8Btrd2HqupQm343cA3wQuDm9gD4yAx9SJLGoCtMqupOYGLErLNGtC3gkhnWswPYMaI+CZw2ov69UX1IksbDT8BLkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknq1h0mSVYk+XqSP2qv1yfZm2R/kuuTHNPqx7bXU23+uqF1XNrqDyQ5Z6i+udWmkmwfqo/sQ5I0HouxZ/JeYN/Q648Cl1fVBuAJ4OJWvxh4oqpeCVze2pFkI3AB8GpgM/DxFlArgI8B5wIbgQtb29n6kCSNQVeYJFkL/F3gE+11gDcBN7YmO4Hz2/SW9po2/6zWfgtwXVX9qKq+DUwBZ7THVFU9WFU/Bq4DtszRhyRpDHr3TH4P+DfAX7bXJwLfr6qn2+sDwJo2vQZ4GKDNf7K1/2n9sGVmqs/WxzMk2ZZkMsnkwYMHF/o1SpLmsOAwSfIW4PGqun24PKJpzTFvserPLlZdVVUTVTWxevXqUU0kSYtgZceybwDemuQ84AXAcQz2VI5PsrLtOawFHmntDwCnAAeSrAReChwaqk8bXmZU/buz9CFJGoMF75lU1aVVtbaq1jE4gf7Fqnon8CXgba3ZVuCmNr2rvabN/2JVVatf0K72Wg9sAL4G3AZsaFduHdP62NWWmakPSdIYLMXnTD4AvD/JFIPzG1e3+tXAia3+fmA7QFXdC9wA3Ad8Hrikqn7S9jreA+xmcLXYDa3tbH1IksYggz/0f/ZNTEzU5OTkgpb91N6HZpz3jjNPXeiQJOmIl+T2qpqYq52fgJckdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3QwTSVI3w0SS1M0wkSR1M0wkSd0ME0lSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLUzTCRJHUzTCRJ3QwTSVI3w0SS1M0wkSR1M0wkSd0ME0lStwWHSZJTknwpyb4k9yZ5b6ufkGRPkv3teVWrJ8kVSaaS3JXk9KF1bW3t9yfZOlR/bZK72zJXJMlsfUiSxqNnz+Rp4F9V1d8CNgGXJNkIbAduqaoNwC3tNcC5wIb22AZcCYNgAC4DzgTOAC4bCocrW9vp5Ta3+kx9SJLGYMFhUlWPVtUdbfopYB+wBtgC7GzNdgLnt+ktwLU1cCtwfJKTgXOAPVV1qKqeAPYAm9u846rqq1VVwLWHrWtUH5KkMViUcyZJ1gGvAfYCL6+qR2EQOMDLWrM1wMNDix1otdnqB0bUmaUPSdIYdIdJkp8H/hB4X1X9YLamI2q1gPp8xrYtyWSSyYMHD85nUUnSPHSFSZKfYxAkn6yqz7TyY+0QFe358VY/AJwytPha4JE56mtH1Gfr4xmq6qqqmqiqidWrVy/si5Qkzannaq4AVwP7qup3h2btAqavyNoK3DRUv6hd1bUJeLIdotoNnJ1kVTvxfjawu817Ksmm1tdFh61rVB+SpDFY2bHsG4B/Atyd5M5W+7fAR4AbklwMPAS8vc37HHAeMAX8EHgXQFUdSvJh4LbW7kNVdahNvxu4BnghcHN7MEsfkqQxWHCYVNVXGH1eA+CsEe0LuGSGde0AdoyoTwKnjah/b1QfkqTx8BPwkqRuhokkqZthIknq1nMCXsCn9j40sv6OM09d5pFI0vi4ZyJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuhkmkqRuhokkqZthIknqZphIkroZJpKkboaJJKmbYSJJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEiSuvlve5eI/85X0vOJeyaSpG5HdZgk2ZzkgSRTSbaPezyS9Hx11IZJkhXAx4BzgY3AhUk2jndUkvT8dDSfMzkDmKqqBwGSXAdsAe4b66jm4LkUST+LjuYwWQM8PPT6AHDmmMbSbaaQmS9DSdI4HM1hkhG1ekaDZBuwrb38v0keWGBfJwHfXeCyS+lZ43rnmAZymKNmex0hHNf8OK7nbjHG9NefS6OjOUwOAKcMvV4LPDLcoKquAq7q7SjJZFVN9K5nsTmu+XFc8+O45udIHNdyjumoPQEP3AZsSLI+yTHABcCuMY9Jkp6Xjto9k6p6Osl7gN3ACmBHVd075mFJ0vPSURsmAFX1OeBzy9BV96GyJeK45sdxzY/jmp8jcVzLNqZU1dytJEmaxdF8zkSSdIQwTOYwzlu2JDklyZeS7Etyb5L3tvpvJ/mTJHe2x3lDy1zaxvpAknOWcGzfSXJ363+y1U5IsifJ/va8qtWT5Io2rruSnL4E43nV0Pa4M8kPkrxvHNsqyY4kjye5Z6g2722TZGtrvz/J1iUa1+8kub/1/dkkx7f6uiR/PrTdfn9omde27/1UG/uoy/R7xzXv79ti/6zOMK7rh8b0nSR3tvpybq+Zfi+M9z1WVT5meDA4sf8t4BXAMcA3gI3L2P/JwOlt+iXANxncOua3gX89ov3GNsZjgfVt7CuWaGzfAU46rPbvge1tejvw0TZ9HnAzg88GbQL2LsP37U8ZXB+/7NsKeCNwOnDPQrcNcALwYHte1aZXLcG4zgZWtumPDo1r3XC7w9bzNeD1bcw3A+cuwbjm9X1bip/VUeM6bP5/BH5rDNtrpt8LY32PuWcyu5/esqWqfgxM37JlWVTVo1V1R5t+CtjH4JP/M9kCXFdVP6qqbwNTDL6G5bIF2NmmdwLnD9WvrYFbgeOTnLyE4zgL+FZV/fEsbZZsW1XVl4FDI/qbz7Y5B9hTVYeq6glgD7B5scdVVV+oqqfby1sZfF5rRm1sx1XVV2vwG+naoa9l0cY1i5m+b4v+szrbuNrexT8CPj3bOpZoe830e2Gs7zHDZHajbtky2y/zJZNkHfAaYG8rvaftsu6Y3p1lecdbwBeS3J7BnQYAXl5Vj8LgDQ+8bAzjgsFnjoZ/yMe9rWD+22Yc771fZ/AX7LT1Sb6e5H8l+dVWW9PGshzjms/3bbm3168Cj1XV/qHasm+vw34vjPU9ZpjMbs5btizLIJKfB/4QeF9V/QC4EvgbwN8GHmWwuw3LO943VNXpDO7afEmSN87SdtnGlcEHWN8K/M9WOhK21WxmGseyji/JB4GngU+20qPAqVX1GuD9wKeSHLeM45rv9225v58X8sw/WJZ9e434vTBj0xnGsKhjM0xmN+ctW5Zakp9j8Ib5ZFV9BqCqHquqn1TVXwL/jb86PLNs462qR9rz48Bn2xgemz581Z4fX+5xMQi3O6rqsTa+sW+rZr7bZtnG1068vgV4ZzsUQzuM9L02fTuD8xG/2MY1fChsSca1gO/bcm6vlcA/AK4fGu+ybq9RvxcY83vMMJndWG/Z0o7LXg3sq6rfHaoPn2/4+8D01Sa7gAuSHJtkPbCBwcm/xR7Xi5O8ZHqawUnce1r/01eEbAVuGhrXRe2qkk3Ak9O740vgGX8xjntbDZnvttkNnJ1kVTvEc3arLaokm4EPAG+tqh8O1Vdn8D+DSPIKBtvnwTa2p5Jsau/Pi4a+lsUc13y/b8v5s/pm4P6q+unhq+XcXjP9XmDc77GeqwqeDw8GV0J8k8FfGh9c5r5/hcFu513Ane1xHvAHwN2tvgs4eWiZD7axPkDnVSOzjOsVDK6W+QZw7/R2AU4EbgH2t+cTWj0M/pHZt9q4J5ZoXC8Cvge8dKi27NuKQZg9CvwFg7/+Ll7ItmFwDmOqPd61ROOaYnDcfPr99fut7T9s39tvAHcAf29oPRMMfrl/C/gvtA8/L/K45v19W+yf1VHjavVrgN84rO1ybq+Zfi+M9T3mJ+AlSd08zCVJ6maYSJK6GSaSpG6GiSSpm2EiSepmmEhjlORDSd487nFIvbw0WBqTJCuq6ifjHoe0GNwzkZZABv/f4v4kO9vNCm9M8qIM/gfGbyX5CvD2JNckeVtb5nVJ/k+SbyT5WpKXJFmRwf8cua2t55+N+UuTRjJMpKXzKuCqqvol4AfAP2/1/1dVv1JV1003bLcAuR54b1X9MoNbdvw5g0+DP1lVrwNeB/zTdhsR6YhimEhL5+Gq+t9t+n8wuA0GDN0gcMirgEer6jaAqvpBDf7PyNkM7qt0J4PbjJ/I4L5P0hFl5bgHIP0MO/yE5PTrPxvRNiPaT9d/s6oW/SaP0mJyz0RaOqcmeX2bvhD4yixt7wd+IcnrANr5kpUM7uL67nbLcZL8YrtTs3REMUykpbMP2JrkLgb/Z/vKmRrW4F/N/mPgPyf5BoN/ofoC4BPAfcAdSe4B/iseUdARyEuDpSXQ/p3qH1XVaWMeirQs3DORJHVzz0SS1M09E0lSN8NEktTNMJEkdTNMJEndDBNJUjfDRJLU7f8Dwib9/Yqo+ggAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Target값인 Price칼럼의 데이터 분포 확인\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "y_train_df=mercari_df['price']\n",
    "\n",
    "plt.figure(figsize=(6,4))\n",
    "sns.distplot(y_train_df, kde=False)\n",
    "\n",
    "# price값이 비교적 적은 가격을 가진 데이터 값에 왜곡되어 있음\n",
    "# 회귀 문제에서 정규 분포도는 매우 중요하기 때문에 정규성을 갖기 위해 log 변환을 수행"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2233561e8d0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEKCAYAAADuEgmxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGXBJREFUeJzt3X+wX3V95/Hnq0HwR4uARJcmsME1souMXfEK7NrtUKMQrGP4A7bBVlLLbHYtWt0fo9DuLLMqMzjtlOqOMpslKcEqgUVdMjY0zYCs6678CKD81HKLFK6giQYRdZUNvveP7+fql5vvzT35fpN8b8LzMfOde877fM75fr4ZuK/7OefzPSdVhSRJXfzSuDsgSTpwGBqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdHTLuDuxtRx99dC1ZsmTc3ZCkA8qdd9753apaOFe7gy40lixZwtatW8fdDUk6oCT5+y7tPD0lSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSepszm+EJ1kHvA3YVlUn9dXfC7wH2An8VVV9oNUvBi4AngX+sKo2t/py4GPAAuDKqrqs1Y8HNgBHAXcB76yqZ5IcBlwNvB74HvDbVfXI3vjQGr/P3PborNvecepx+7EnkvZEl5HGVcDy/kKS3wRWAK+tqtcAf9rqJwIrgde0fT6ZZEGSBcAngLOAE4HzWluAjwKXV9VS4El6gUP7+WRVvQq4vLWTJI3RnKFRVV8Cdswovxu4rKp+2tpsa/UVwIaq+mlVfROYBE5pr8mqeriqnqE3sliRJMCbgOvb/uuBs/uOtb4tXw8sa+0lSWMy7DWNVwP/IsltSf5nkje0+iLgsb52U602W/1lwPeraueM+nOO1bY/1dpLksZk2LvcHgIcCZwGvAG4LskrgUEjgWJwONVu2jPHtudIshpYDXDccZ4Pl6R9ZdiRxhTwueq5HfgZcHSrH9vXbjHw+G7q3wWOSHLIjDr9+7TtL2XX02QAVNWaqpqoqomFC+e8HbwkaUjDhsb/oHctgiSvBg6lFwAbgZVJDmuzopYCtwN3AEuTHJ/kUHoXyzdWVQFfBM5px10F3NCWN7Z12vabW3tJ0ph0mXJ7DXA6cHSSKeASYB2wLsl9wDPAqvYL/f4k1wEP0JuKe2FVPduO8x5gM70pt+uq6v72Fh8ENiT5CHA3sLbV1wKfSjJJb4Sxci98XknSCOYMjao6b5ZNvztL+0uBSwfUNwGbBtQfpje7amb9J8C5c/VPkrT/+I1wSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzuYMjSTrkmxrT+mbue0/JKkkR7f1JPl4kskk9yQ5ua/tqiQPtdeqvvrrk9zb9vl4krT6UUm2tPZbkhy5dz6yJGlYXUYaVwHLZxaTHAu8BXi0r3wWveeCLwVWA1e0tkfRe0zsqfSe0ndJXwhc0dpO7zf9XhcBN1XVUuCmti5JGqM5Q6OqvkTvGd0zXQ58AKi+2grg6uq5FTgiyTHAmcCWqtpRVU8CW4DlbdvhVfWV9ozxq4Gz+461vi2v76tLksZkqGsaSd4OfKuqvjZj0yLgsb71qVbbXX1qQB3gFVX1BED7+fJh+ipJ2nsO2dMdkrwY+GPgjEGbB9RqiPqe9mk1vVNcHHfccXu6uySpo2FGGv8IOB74WpJHgMXAXUn+Ab2RwrF9bRcDj89RXzygDvCddvqK9nPbbB2qqjVVNVFVEwsXLhziI0mSutjj0Kiqe6vq5VW1pKqW0PvFf3JVfRvYCJzfZlGdBjzVTi1tBs5IcmS7AH4GsLltezrJaW3W1PnADe2tNgLTs6xW9dUlSWPSZcrtNcBXgBOSTCW5YDfNNwEPA5PAfwP+AKCqdgAfBu5orw+1GsC7gSvbPn8H3NjqlwFvSfIQvVlal+3ZR5Mk7W1zXtOoqvPm2L6kb7mAC2dptw5YN6C+FThpQP17wLK5+idJ2n/8RrgkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSepsj5+nIR3oPnPbowPr7zjVZ7FIc3GkIUnqzNCQJHVmaEiSOvOahg54XqOQ9h9HGpKkzro87nVdkm1J7uur/UmSrye5J8nnkxzRt+3iJJNJvpHkzL768labTHJRX/34JLcleSjJtUkObfXD2vpk275kb31oSdJwuow0rgKWz6htAU6qqtcCfwtcDJDkRGAl8Jq2zyeTLEiyAPgEcBZwInBeawvwUeDyqloKPAlMP4P8AuDJqnoVcHlrJ0kaozlDo6q+BOyYUfubqtrZVm8FFrflFcCGqvppVX0TmAROaa/Jqnq4qp4BNgArkgR4E3B92389cHbfsda35euBZa29JGlM9sY1jd8HbmzLi4DH+rZNtdps9ZcB3+8LoOn6c47Vtj/V2u8iyeokW5Ns3b59+8gfSJI02EihkeSPgZ3Ap6dLA5rVEPXdHWvXYtWaqpqoqomFCxfuvtOSpKENPeU2ySrgbcCyqpr+ZT4FHNvXbDHweFseVP8ucESSQ9poor/99LGmkhwCvJQZp8l0cHIKrTR/DTXSSLIc+CDw9qr6cd+mjcDKNvPpeGApcDtwB7C0zZQ6lN7F8o0tbL4InNP2XwXc0HesVW35HODmvnCSJI3BnCONJNcApwNHJ5kCLqE3W+owYEu7Nn1rVf2bqro/yXXAA/ROW11YVc+247wH2AwsANZV1f3tLT4IbEjyEeBuYG2rrwU+lWSS3ghj5V74vNKsHOFIc5szNKrqvAHltQNq0+0vBS4dUN8EbBpQf5je7KqZ9Z8A587VP0nS/uM3wiVJnRkakqTODA1JUmfe5VYHjNkuVEvafxxpSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnc4ZGknVJtiW5r692VJItSR5qP49s9ST5eJLJJPckOblvn1Wt/UPt+eLT9dcnubft8/G0RwHO9h6SpPHpMtK4Clg+o3YRcFNVLQVuausAZ9F7LvhSYDVwBfQCgN5jYk+l95S+S/pC4IrWdnq/5XO8hyRpTOYMjar6Er1ndPdbAaxvy+uBs/vqV1fPrcARSY4BzgS2VNWOqnoS2AIsb9sOr6qvVFUBV8841qD3kCSNybDP03hFVT0BUFVPJHl5qy8CHutrN9Vqu6tPDajv7j2kTnz+hrT37e0L4RlQqyHqe/amyeokW5Ns3b59+57uLknqaNjQ+E47tUT7ua3Vp4Bj+9otBh6fo754QH1377GLqlpTVRNVNbFw4cIhP5IkaS7DhsZGYHoG1Crghr76+W0W1WnAU+0U02bgjCRHtgvgZwCb27ank5zWZk2dP+NYg95DkjQmc17TSHINcDpwdJIperOgLgOuS3IB8Chwbmu+CXgrMAn8GHgXQFXtSPJh4I7W7kNVNX1x/d30Zmi9CLixvdjNe0iSxmTO0Kiq82bZtGxA2wIunOU464B1A+pbgZMG1L836D0kSePjN8IlSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6G/Yut9Lzxmx3y33Hqcft555I4+dIQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqbORQiPJv01yf5L7klyT5IVJjk9yW5KHklyb5NDW9rC2Ptm2L+k7zsWt/o0kZ/bVl7faZJKLRumrJGl0Q4dGkkXAHwITVXUSsABYCXwUuLyqlgJPAhe0XS4AnqyqVwGXt3YkObHt9xpgOfDJJAuSLAA+AZwFnAic19pKksZk1NNThwAvSnII8GLgCeBNwPVt+3rg7La8oq3Tti9LklbfUFU/rapvApPAKe01WVUPV9UzwIbWVpI0JkOHRlV9C/hT4FF6YfEUcCfw/ara2ZpNAYva8iLgsbbvztb+Zf31GfvMVt9FktVJtibZun379mE/kiRpDqOcnjqS3l/+xwO/CryE3qmkmWp6l1m27Wl912LVmqqaqKqJhQsXztV1SdKQRjk99Wbgm1W1var+H/A54J8DR7TTVQCLgcfb8hRwLEDb/lJgR399xj6z1SVJYzJKaDwKnJbkxe3axDLgAeCLwDmtzSrghra8sa3Ttt9cVdXqK9vsquOBpcDtwB3A0jYb61B6F8s3jtBfSdKIhn6eRlXdluR64C5gJ3A3sAb4K2BDko+02tq2y1rgU0km6Y0wVrbj3J/kOnqBsxO4sKqeBUjyHmAzvZlZ66rq/mH7K0kaXXp/7B88JiYmauvWrePuhuYw24ONDiQ+hEkHkyR3VtXEXO38RrgkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnQz+5DyDJEcCVwElAAb8PfAO4FlgCPAL8y6p6sj0S9mPAW4EfA79XVXe146wC/mM77Eeqan2rvx64CngRsAl4Xx1sT43SAWu2B0n5cCYdzEYdaXwM+Ouq+sfArwEPAhcBN1XVUuCmtg5wFr3nfy8FVgNXACQ5CrgEOBU4BbgkyZFtnyta2+n9lo/YX0nSCIYeaSQ5HPgN4PcAquoZ4JkkK4DTW7P1wC3AB4EVwNVtpHBrkiOSHNPabqmqHe24W4DlSW4BDq+qr7T61cDZwI3D9ln738HwWFdJvzDKSOOVwHbgL5LcneTKJC8BXlFVTwC0ny9v7RcBj/XtP9Vqu6tPDahLksZklNA4BDgZuKKqXgf8iF+cihokA2o1RH3XAyerk2xNsnX79u2777UkaWijXAifAqaq6ra2fj290PhOkmOq6ol2+mlbX/tj+/ZfDDze6qfPqN/S6osHtN9FVa0B1gBMTEx4oVzzkhfOdTAYeqRRVd8GHktyQistAx4ANgKrWm0VcENb3gicn57TgKfa6avNwBlJjmwXwM8ANrdtTyc5rc28Or/vWJKkMRhpyi3wXuDTSQ4FHgbeRS+IrktyAfAocG5ru4nedNtJelNu3wVQVTuSfBi4o7X70PRFceDd/GLK7Y14EVySxmqk0KiqrwITAzYtG9C2gAtnOc46YN2A+lZ63wGRJM0DfiNcktSZoSFJ6szQkCR1ZmhIkjobdfaUBHi7EOn5wpGGJKkzRxrSXuaoSwczRxqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM5GDo0kC5LcneQLbf34JLcleSjJte2pfiQ5rK1Ptu1L+o5xcat/I8mZffXlrTaZ5KJR+ypJGs3eGGm8D3iwb/2jwOVVtRR4Erig1S8AnqyqVwGXt3YkORFYCbwGWA58sgXRAuATwFnAicB5ra0kaUxGCo0ki4HfAq5s6wHeBFzfmqwHzm7LK9o6bfuy1n4FsKGqflpV36T3DPFT2muyqh6uqmeADa2tJGlMRh1p/DnwAeBnbf1lwPeramdbnwIWteVFwGMAbftTrf3P6zP2ma0uSRqToUMjyduAbVV1Z395QNOaY9ue1gf1ZXWSrUm2bt++fTe9liSNYpSRxhuBtyd5hN6pozfRG3kckWT6luuLgcfb8hRwLEDb/lJgR399xj6z1XdRVWuqaqKqJhYuXDjCR5Ik7c7QoVFVF1fV4qpaQu9C9s1V9TvAF4FzWrNVwA1teWNbp22/uaqq1Ve22VXHA0uB24E7gKVtNtah7T02DttfSdLo9sVDmD4IbEjyEeBuYG2rrwU+lWSS3ghjJUBV3Z/kOuABYCdwYVU9C5DkPcBmYAGwrqru3wf9lSR1tFdCo6puAW5pyw/Tm/k0s81PgHNn2f9S4NIB9U3Apr3RR0nS6PxGuCSpM0NDktSZoSFJ6szQkCR1ZmhIkjrbF1NudRD7zG2PjrsLksbI0NBAhoOkQTw9JUnqzNCQJHXm6SlpzIY5FfiOU4/bBz2R5uZIQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzpxyKx2AZpum61Rc7WtDjzSSHJvki0keTHJ/kve1+lFJtiR5qP08stWT5ONJJpPck+TkvmOtau0fSrKqr/76JPe2fT6eJKN8WEnSaEY5PbUT+PdV9U+A04ALk5wIXATcVFVLgZvaOsBZwNL2Wg1cAb2QAS4BTqX3mNhLpoOmtVndt9/yEforSRrR0KFRVU9U1V1t+WngQWARsAJY35qtB85uyyuAq6vnVuCIJMcAZwJbqmpHVT0JbAGWt22HV9VXqqqAq/uOJUkag71yITzJEuB1wG3AK6rqCegFC/Dy1mwR8FjfblOttrv61ID6oPdfnWRrkq3bt28f9eNIkmYxcmgk+WXgs8D7q+oHu2s6oFZD1HctVq2pqomqmli4cOFcXZYkDWmk2VNJXkAvMD5dVZ9r5e8kOaaqnminmLa1+hRwbN/ui4HHW/30GfVbWn3xgPYagrNtJO0No8yeCrAWeLCq/qxv00ZgegbUKuCGvvr5bRbVacBT7fTVZuCMJEe2C+BnAJvbtqeTnNbe6/y+Y0mSxmCUkcYbgXcC9yb5aqv9EXAZcF2SC4BHgXPbtk3AW4FJ4MfAuwCqakeSDwN3tHYfqqodbfndwFXAi4Ab20uSNCZDh0ZVfZnB1x0Alg1oX8CFsxxrHbBuQH0rcNKwfdTcfKyrpD3hbUQkSZ0ZGpKkzrz3lHQQcZac9jVDQ3oeMEy0t3h6SpLUmaEhSerM0JAkdWZoSJI6MzQkSZ05e0p6HnNWlfaUIw1JUmeONCTtwhGIZuNIQ5LUmSMNSZ05ApEjDUlSZ440JI3MEcjzx7wPjSTLgY8BC4Arq+qyMXdJUkeGycFnXodGkgXAJ4C3AFPAHUk2VtUD4+2ZpFEYJgeueR0awCnAZFU9DJBkA7ACMDSkg9Awjx82aPav+R4ai4DH+tangFP31Zv514904NnXz7n3///nmu+hkQG12qVRshpY3VZ/mOQbQ77f0cB3ZxZ/Z8iD7QMD+zeP2L/R2L/R7JP+7cX//+f7v98/7NJovofGFHBs3/pi4PGZjapqDbBm1DdLsrWqJkY9zr5i/0Zj/0Zj/0Yz3/vX1Xz/nsYdwNIkxyc5FFgJbBxznyTpeWtejzSqameS9wCb6U25XVdV94+5W5L0vDWvQwOgqjYBm/bT2418imsfs3+jsX+jsX+jme/96yRVu1xXliRpoPl+TUOSNI8YGk2S5Um+kWQyyUXj7k+/JOuSbEty37j7MkiSY5N8McmDSe5P8r5x96lfkhcmuT3J11r//vO4+zRIkgVJ7k7yhXH3ZaYkjyS5N8lXk2wdd39mSnJEkuuTfL39d/jPxt2naUlOaP9u068fJHn/uPs1LE9P8fPblfwtfbcrAc6bL7crSfIbwA+Bq6vqpHH3Z6YkxwDHVNVdSX4FuBM4ex79+wV4SVX9MMkLgC8D76uqW8fctedI8u+ACeDwqnrbuPvTL8kjwERVzcvvGSRZD/yvqrqyzbR8cVV9f9z9mqn9rvkWcGpV/f24+zMMRxo9P79dSVU9A0zfrmReqKovATvG3Y/ZVNUTVXVXW34aeJDet/nnher5YVt9QXvNq7+WkiwGfgu4ctx9OdAkORz4DWAtQFU9Mx8Do1kG/N2BGhhgaEwbdLuSefNL70CSZAnwOuC28fbkudqpn68C24AtVTWv+gf8OfAB4Gfj7sgsCvibJHe2OzDMJ68EtgN/0U7vXZnkJePu1CxWAteMuxOjMDR6Ot2uRLuX5JeBzwLvr6ofjLs//arq2ar6p/TuKnBKknlzmi/J24BtVXXnuPuyG2+sqpOBs4AL2ynT+eIQ4GTgiqp6HfAjYF5dlwRop83eDvz3cfdlFIZGT6fblWh27VrBZ4FPV9Xnxt2f2bTTFrcAy8fclX5vBN7erhtsAN6U5C/H26XnqqrH289twOfpndKdL6aAqb7R4/X0QmS+OQu4q6q+M+6OjMLQ6PF2JSNoF5rXAg9W1Z+Nuz8zJVmY5Ii2/CLgzcDXx9urX6iqi6tqcVUtofff3s1V9btj7tbPJXlJm+BAO+1zBjBvZvJV1beBx5Kc0ErLmJ+PTziPA/zUFBwA3wjfH+b77UqSXAOcDhydZAq4pKrWjrdXz/FG4J3Ave26AcAftW/zzwfHAOvbzJVfAq6rqnk3rXUeewXw+d7fBhwCfKaq/nq8XdrFe4FPtz/6HgbeNeb+PEeSF9Obnfmvx92XUTnlVpLUmaenJEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIe0HST6U5M3j7oc0KqfcSvtYkgVV9ey4+yHtDY40pBEkWdKe4bA+yT3tmQ4vbs+f+E9Jvgycm+SqJOe0fd6Q5P+053vcnuRX2g0V/yTJHe04B/yXwHRwMjSk0Z0ArKmq1wI/AP6g1X9SVb9eVRumG7ZvLF9L73kev0bvlib/F7gAeKqq3gC8AfhXSY7fnx9C6sLQkEb3WFX977b8l8Cvt+VrB7Q9AXiiqu4AqKofVNVOevdzOr/dhuU24GXA0n3bbWnPee8paXQzLwxOr/9oQNsMaD9df29Vbd6bHZP2Nkca0uiO63sm9Xn0Hic7m68Dv5rkDQDtesYh9G6W+e52i3mSvHoeP0hIz2OGhjS6B4FVSe4BjgKumK1he5zwbwP/JcnXgC3AC+k95vUB4K4k9wH/Fc8EaB5yyq00gvZ42y9U1bx5EqC0LznSkCR15khDktSZIw1JUmeGhiSpM0NDktSZoSFJ6szQkCR1ZmhIkjr7/3azHrjDhIhDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# log변환 후, price 칼럼의 데이터 분포\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "y_train_df=np.log1p(y_train_df)\n",
    "sns.distplot(y_train_df, kde=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2.397895\n",
       "1    3.970292\n",
       "2    2.397895\n",
       "Name: price, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mercari_df['price']=np.log1p(mercari_df['price'])\n",
    "mercari_df['price'].head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shipping값 유형: \n",
      " 0    819435\n",
      "1    663100\n",
      "Name: shipping, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Shipping(배송비 무료 여부): 0 - 판매자 부담 / 1 - 구매자 부담\n",
    "print('shipping값 유형: \\n', mercari_df['shipping'].value_counts())\n",
    "    #비교적 균일"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "item_condition_id 값 유형: \n",
      " 1    640549\n",
      "3    432161\n",
      "2    375479\n",
      "4     31962\n",
      "5      2384\n",
      "Name: item_condition_id, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Item_condition_id (판매자가 제공하는 제품 상태): 0 / 1 / 2 / 3 / 4 / 5\n",
    "print('item_condition_id 값 유형: \\n', mercari_df['item_condition_id'].value_counts())\n",
    "\n",
    "    # 1,2,3 값이 주를 이룸."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "82489"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Item_description (제품에 대한 간략한 설명)\n",
    "boolean_cond=mercari_df['item_description']=='No description yet'\n",
    "mercari_df[boolean_cond]['item_description'].count()\n",
    "\n",
    "# null값은 아니지만, 설명이 없다는 것은 사실상 null값."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                    Men/Tops/T-shirts\n",
       "1    Electronics/Computers & Tablets/Components & P...\n",
       "2                          Women/Tops & Blouses/Blouse\n",
       "3                   Home/Home Décor/Home Décor Accents\n",
       "4                              Women/Jewelry/Necklaces\n",
       "Name: category_name, dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Category_name (분류명): 대분류/중분류/소분류\n",
    "mercari_df['category_name'][:5]\n",
    "\n",
    "# Men/Tops/T-shirts는 대분류 :Men / 중분류:Tops / 소분류:T-shirts로 이루어져 있음"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "대분류 유형 Women                     664385\n",
      "Beauty                    207828\n",
      "Kids                      171689\n",
      "Electronics               122690\n",
      "Men                        93680\n",
      "Home                       67871\n",
      "Vintage & Collectibles     46530\n",
      "Other                      45351\n",
      "Handmade                   30842\n",
      "Sports & Outdoors          25342\n",
      "Other_Null                  6327\n",
      "Name: cat_dae, dtype: int64\n",
      "중분류  114\n",
      "소분류  871\n"
     ]
    }
   ],
   "source": [
    "# apply lambda에서 호출되는 대, 중, 소 분할 함수 생성. 리스트로 반환\n",
    "\n",
    "def split_cat(category_name):\n",
    "    try:\n",
    "        return category_name.split('/')\n",
    "    except:\n",
    "        return ['Other_Null','Other_Null','Other_Null']\n",
    "\n",
    "    \n",
    "# 정의한 함수를 mercari_df의 category_name 칼럼에 적용 및 분리\n",
    "mercari_df['cat_dae'],mercari_df['cat_jung'],mercari_df['cat_so']=zip(*mercari_df['category_name'].apply(lambda x:split_cat(x)))\n",
    "                    # zip(*iterable)함수 : 동일한 개수로 이루어진 리스트를 각 인덱스에 따라 묶어주는 함수\n",
    "    \n",
    "# 대분류만 값의 유형과 건수 확인, 중,소는 유형이 많으므로 분류 개수만 추출\n",
    "print('대분류 유형', mercari_df['cat_dae'].value_counts())\n",
    "print('중분류 ', mercari_df['cat_jung'].nunique())\n",
    "print('소분류 ', mercari_df['cat_so'].nunique())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train_id             0\n",
       "name                 0\n",
       "item_condition_id    0\n",
       "category_name        0\n",
       "brand_name           0\n",
       "price                0\n",
       "shipping             0\n",
       "item_description     0\n",
       "cat_dae              0\n",
       "cat_jung             0\n",
       "cat_so               0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  Null 값이 존재하는 모든 칼럼에 'Other Null' 처리.\n",
    "mercari_df['brand_name']=mercari_df['brand_name'].fillna(value='Other_Null')\n",
    "mercari_df['category_name']=mercari_df['category_name'].fillna(value='Other_Null')\n",
    "mercari_df['item_description']=mercari_df['item_description'].fillna(value='Other_Null')\n",
    "mercari_df.isnull().sum()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 회귀 모델을 기반으로 price를 예측 할 것이기 때문에 문자열 컬럼을 원-핫 인코딩 또는 피처 벡터화를 진행 \n",
    "- 회귀 모델을 위한 데이터 셋 정리\n",
    "    - 1) 기본적으로 원-핫 인코딩을 적용\n",
    "    - 2) 짧은 텍스트:Count기반 벡터화 적용\n",
    "    - 3) 긴 텍스트:TF-IDF기반 벡터화 적용\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 피처 인코딩과 피처 벡터화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "brand name의 유형 건수 4810\n",
      "brand name sample 10건:  Other_Null           632682\n",
      "PINK                  54088\n",
      "Nike                  54043\n",
      "Victoria's Secret     48036\n",
      "LuLaRoe               31024\n",
      "Apple                 17322\n",
      "FOREVER 21            15186\n",
      "Nintendo              15007\n",
      "Lululemon             14558\n",
      "Michael Kors          13928\n",
      "Name: brand_name, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print('brand name의 유형 건수',mercari_df['brand_name'].nunique())\n",
    "print('brand name sample 10건: ', mercari_df['brand_name'].value_counts()[:10])\n",
    "\n",
    "# 대부분 명료한 문자열이며 종류는 4810개\n",
    "# --> 원-핫 인코딩으로 변환"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name의 종류 개수:  1225273\n",
      "name의 sample 10건:  0       MLB Cincinnati Reds T Shirt Size XL\n",
      "1          Razer BlackWidow Chroma Keyboard\n",
      "2                            AVA-VIV Blouse\n",
      "3                     Leather Horse Statues\n",
      "4                      24K GOLD plated rose\n",
      "5          Bundled items requested for Ruie\n",
      "6        Acacia pacific tides santorini top\n",
      "7      Girls cheer and tumbling bundle of 7\n",
      "8                     Girls Nike Pro shorts\n",
      "9    Porcelain clown doll checker pants VTG\n",
      "Name: name, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print('name의 종류 개수: ', mercari_df['name'].nunique())\n",
    "print('name의 sample 10건: ', mercari_df['name'][:10])\n",
    "\n",
    "# 속성의 종류가 매우 많고, 단어위주의 텍스트 형태\n",
    "# --> Count기반으로 피처 벡터화 변환을 적용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "item_description 평균 문자열 크기: 145.7113889385411\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0                                                                                                                                                                              No description yet\n",
       "1    This keyboard is in great condition and works like it came out of the box. All of the ports are tested and work perfectly. The lights are customizable via the Razer Synapse app on your PC.\n",
       "Name: item_description, dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('max_colwidth',200)\n",
    "\n",
    "print('item_description 평균 문자열 크기:', mercari_df['item_description'].str.len().mean())\n",
    "mercari_df['item_description'][:2]\n",
    "\n",
    "# 평균 문자열이 145자로 비교적 크므로 TF-IDF 변환"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 이외의 컬럼\n",
    "- category_name 컬럼은 이전 전처리를 통해 대,중,소로 분리 했으므로 원-핫 인코딩 적용\n",
    "- shipping 컬럼은 0과 1두가지 유형을 가지고 있으므로 원-핫 인코딩 적용\n",
    "- item_condition_id 1,2,3,4,5의 다섯 가지 유형의 값을 가지고 있으므로 원-핫 인코딩 적용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name 벡터화 shape:  (1482535, 105757)\n"
     ]
    }
   ],
   "source": [
    "# Name count 기반 벡터화. \n",
    "cnt_vec=CountVectorizer()\n",
    "X_name=cnt_vec.fit_transform(mercari_df.name)\n",
    "print('name 벡터화 shape: ', X_name.shape)\n",
    "\n",
    "# item_description  Tf-IDF 피처 벡터화\n",
    "tfidf_descp=TfidfVectorizer(max_features=50000, ngram_range=(1,3), stop_words='english')\n",
    "X_dscrp=tfidf_descp.fit_transform(mercari_df['item_description'])\n",
    "print('item_description 벡터화 shape: ', X_dscrp.shape)\n",
    "\n",
    "# 변환된 결과는 희소 행렬 형태"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 위의 결과가 희소행렬 형태이므로 \n",
    "- LabelBinarizer를 이용해 원-핫 인코딩 피쳐들을 희소 행렬로 변환 후 피처 벡터화된 희소행렬들과 결합\n",
    "\n",
    "    - 사이킷런은 원-핫 인코딩을 위해 OneHotEncoder / LabelBinarizer를 지원\n",
    "    - 이중 LabelBinarizer는 희소행렬 형태의 원-핫 인코딩을 지원"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LabelBinarizer 이해\n",
    "from sklearn.preprocessing import LabelBianrizer\n",
    "\n",
    "feature = np.array([['Texas'],['California'],['Texas'],['Delaware'],['Texas']])\n",
    "one_hot = LabelBianrizer()\n",
    "print(one_hot.fit_transform(feature))\n",
    "print(one_hot.classes_)\n",
    "\n",
    "#LabelBinarizer(sparse_output =True)옵션을 통해 희소 행렬 형태로 반환 \n",
    "one_hot = LabelBinarizer(sparse_output=True)\n",
    "print(one_hot.fit_transform(feature))\n",
    "\n",
    "# 희소 행렬은 1인 값의 위치만 저장함"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LabelBinarizer를 통해 희소 행렬 형태로 원-핫 인코딩 변환"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelBianrizer\n",
    "\n",
    "# brand_name, item_condition_id, shipping 피처들을 희소행렬 one-hot 인코딩 변환\n",
    "lb_brand_name = LabelBinarizer(sparse_output=True)\n",
    "X_brand = lb_brand_name.fit_transform(mercari_df['brand_name'])\n",
    "\n",
    "lb_item_cond_id = LabelBinarizer(sparse_output=True)\n",
    "X_item_cond_id = lb_item_cond_id.fit_transform(mercari_df['item_cond_id'])\n",
    "\n",
    "\n",
    "lb_shipping = LabelBinarizer(sparse_output=True)\n",
    "X_shipping = lb_shipping.fit_transform(mercari_df['shipping'])\n",
    "\n",
    "# 대,중,소 피처들을 희소 행렬 one-hot 인코딩 변환\n",
    "lb_cat_dae = LabelBinarizer(sparse_output=True)\n",
    "X_cat_dae = lb_cat_dae.fit_transform(mercari_df['cat_dae'])\n",
    "\n",
    "lb_cat_jung = LabelBinarizer(sparse_output=True)\n",
    "X_cat_jung = lb_cat_dae.fit_transform(mercari_df['cat_jung'])\n",
    "\n",
    "\n",
    "lb_cat_so = LabelBinarizer(sparse_output=True)\n",
    "X_cat_so = lb_cat_dae.fit_transform(mercari_df['cat_so'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 변환되었는지 확인\n",
    "print(type(X_brand),type(X_item_cond_id),type(X_shipping))\n",
    "print('X_brand shape:{0}, X_item_cond_id:{1}'.format(X_brand.shape, X_item_cond_id.shape))\n",
    "print('X_shipping shape:{0}, X_dae:{1}'.format(X_shipping.shape, X_cat_dae.shape))\n",
    "print('X_cat_jung shape:{0}, X_so:{1}'.format(X_cat_jung.shape, X_cat_so.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 변환된 데이터셋을 hstack()을 이용해 모두 결합\n",
    "\n",
    "from scipy.sparse import hstack\n",
    "import gc\n",
    "\n",
    "sparse_matrix_list = (X_name, X_dscrp, X_brand, X_item_cond_id, X_shipping, X_cat_dae, X_cat_jung, X_cat_so)\n",
    "\n",
    "X_feature_sparse=hstack(sparse_matrix_list).tocsr()\n",
    "print(type(X_feature_sparse),X_feature_sparse.shape)\n",
    "\n",
    "# 사용목적이 끝났기때문에 바로 메모리에서 삭제\n",
    "del X_feature_sparse\n",
    "gc.collect()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### X_feature (1482535, 161569) 매우 피쳐의 개수가 많으므 Overfitting을 피하기 위해 릿지 회귀를 사용함.\n",
    "\n",
    "- 릿지 회귀는 규제가 추가된 선형 회귀 ( 계수가 지나치게 커지지 않도록 제한을 걸어 줌)\n",
    "- 릿지 회귀식 : 잔차제곱합(RSS) + 패널티 규제 항\n",
    "- Overfitting을 피하는 방법 중 하나로 알려짐.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RMSLE - 적용할 평가 지표\n",
    "- RMSLE는 RSME와 유사하나 오류 값에 로그를 취해 RMSE를 구하는 것\n",
    "    - 1) 아웃라이어에 Robust하다 :아웃라이어가 있더라도 변동 폭이 크지 않다.\n",
    "    - 2)로그 공식에 의해 상대적 Error를 측정해 준다\n",
    "    - 3)UnderEstimation에 큰 페널티를 부여한다.:예측 값이 실제 값 보다 클 때 보다 예측 값이 실제보다 작을 때\n",
    "더 큰 페널티를 부여한다.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 평가지표 함수 생성\n",
    "\n",
    "def rmsle(y,y_pred):\n",
    "    # underflow, overflow를 막기위해 log가 아닌 log1p로 rmsle계산\n",
    "    return np.sqrt(np.mean(np.power(np.log1p(y)-np.log1p(y_pred),2)))\n",
    "            # price값이 로그변환되어있으므로, 예측도 당연히 로그로 변환한 price값을 예측.        \n",
    "    \n",
    "def evaluate_org_price(y_test,preds):\n",
    "    \n",
    "    # Exponential변환을 통한 실제 값으로의 원복 필요 \n",
    "    preds_exmpm=np.expm1(preds)\n",
    "    y_test_exmpm=np.expm1(y_test)\n",
    "    \n",
    "    rmsle_result=rmsle(y_test_exmpm,preds_exmpm)\n",
    "    \n",
    "    return rmsle_result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 회귀 모델 학습/예측 함수 생성\n",
    "import gc\n",
    "from scipy.sparse import hstack\n",
    "\n",
    "del model_train_predict(model,matrix_list):\n",
    "    \n",
    "    # 희소행렬 결합\n",
    "    X=hstack(matrix_list).tocsr()\n",
    "    \n",
    "    X_train_X_test,y_trin,y_test = train_test_split(X,mercari_df['price'],\n",
    "                                                   test_size=0.2, random_state=156)\n",
    "    #모델 학습 및 예측\n",
    "    model.fit(X_train,y_train)\n",
    "    preds=model.predict(X_test)\n",
    "    \n",
    "    del X,X_train,X_test,y_train\n",
    "    gc.collect()\n",
    "    \n",
    "    return preds,y_test\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ridge 회귀모델 적용\n",
    "\n",
    "linear_model = Ridge(solver='lsqr',fit_intercept=False)\n",
    "\n",
    "sparse_matrix_list=(X_name, X_brand, X_item_cond_id,\n",
    "                   X_shipping, X_cat_dae, X_cat_jung, X_cat_so)\n",
    "\n",
    "# 예측 정확도에 item_description 과 같은 텍스트형 속성이 얼마나 영향을 미치는지 확인\n",
    "\n",
    "linear_preds,y_test=model_train_predict(model=linear_model, matrix_lsit=sparse_matrix_list)\n",
    "print('item description을 제외 하였을 때,' , evaluate_org_price(y_test,linear_preds))\n",
    "sparse_matrix_list=(X_dscrp, X_name, X_brand, X_item_cond_id,\n",
    "                   X_shipping, X_cat_dae, X_cat_jung, X_cat_so)\n",
    "print('item description을 포함 하였을 때,' , evaluate_org_price(y_test,linear_preds))\n",
    "\n",
    "# Itemdescription을 포함했을 때 RMSLE값이 많이 감소한 것을 확인 가능\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LightGBM 회귀 모델 구축과 앙상블을 이용한 최종 예측 평가 모델링\n",
    "\n",
    "from lightgbm import LGBMRegressor\n",
    "parse_matrix_list = (X_dscrp, X_name, X_brand, X_item_cond_id,\n",
    "                   X_shipping, X_cat_dae, X_cat_jung, X_cat_so) \n",
    "\n",
    "lgbm_model=LGBMRegressor(n_estimators=200, learning_rate=0.5, num_leaves=125, random_state=156)\n",
    "lgbm_preds,y_test=model_train_predict(model=lgbm_model, matrix_list=sparse_matrix_list)\n",
    "print('LightGBM rmsle 값:', evaluate_org_price(y_test,lgbm_preds))\n",
    "# Ridge 예측 정확도보다 향상\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 앙상블\n",
    "\n",
    "# LightGBM 예측값에 0.45, Ridge 예측값에에 0.55를 각각 곱한 값으로 예측 결과를 도출\n",
    "    # 0.45와 0.55의 배합 비율은 임의 지정\n",
    "    \n",
    "preds=lgbm_preds*0.45+linear_preds*0,55\n",
    "print('LightGBM과 Ridge를 앙상블한 rmsle 값:', evaluate_org_price(y_test,preds))\n",
    "\n",
    "# 간단한 앙상블 방법으로 예측성능이 개선됌."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}