Activity01.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from textblob import TextBlob\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth',240)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.read_csv('data/airline-tweets.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@FlyHighAirways What @dhepburn said.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@FlyHighAirways plus you've added commercials to the experience... tacky.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@FlyHighAirways I didn't today... Must mean I need to take another trip!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp;amp; they have little recourse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@FlyHighAirways and it's a really big bad thing about it</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                             tweet\n",
       "0                                                                                             @FlyHighAirways What @dhepburn said.\n",
       "1                                                        @FlyHighAirways plus you've added commercials to the experience... tacky.\n",
       "2                                                         @FlyHighAirways I didn't today... Must mean I need to take another trip!\n",
       "3  @FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp; they have little recourse\n",
       "4                                                                         @FlyHighAirways and it's a really big bad thing about it"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['At'] = tweets['tweet'].str.extract(r'^(@\\S+)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>At</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@FlyHighAirways What @dhepburn said.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@FlyHighAirways plus you've added commercials to the experience... tacky.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@FlyHighAirways I didn't today... Must mean I need to take another trip!</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp;amp; they have little recourse</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@FlyHighAirways and it's a really big bad thing about it</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>@FlyHighAirways seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>@FlyHighAirways yes, nearly every time I fly VX this “ear worm” won’t go away :)</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@FlyHighAirways Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>@FlyHighAirways Well, I didn't…but NOW I DO! :-D</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@FlyHighAirways it was amazing, and arrived an hour early. You're too good to me.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                         tweet  \\\n",
       "0                                                                                                         @FlyHighAirways What @dhepburn said.   \n",
       "1                                                                    @FlyHighAirways plus you've added commercials to the experience... tacky.   \n",
       "2                                                                     @FlyHighAirways I didn't today... Must mean I need to take another trip!   \n",
       "3              @FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp; they have little recourse   \n",
       "4                                                                                     @FlyHighAirways and it's a really big bad thing about it   \n",
       "5  @FlyHighAirways seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA   \n",
       "6                                                             @FlyHighAirways yes, nearly every time I fly VX this “ear worm” won’t go away :)   \n",
       "7                                @FlyHighAirways Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP   \n",
       "8                                                                                             @FlyHighAirways Well, I didn't…but NOW I DO! :-D   \n",
       "9                                                            @FlyHighAirways it was amazing, and arrived an hour early. You're too good to me.   \n",
       "\n",
       "                At  \n",
       "0  @FlyHighAirways  \n",
       "1  @FlyHighAirways  \n",
       "2  @FlyHighAirways  \n",
       "3  @FlyHighAirways  \n",
       "4  @FlyHighAirways  \n",
       "5  @FlyHighAirways  \n",
       "6  @FlyHighAirways  \n",
       "7  @FlyHighAirways  \n",
       "8  @FlyHighAirways  \n",
       "9  @FlyHighAirways  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_handles(tweet):\n",
    "    return re.sub(r'@\\S+', '', tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>At</th>\n",
       "      <th>tweet_preprocessed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@FlyHighAirways What @dhepburn said.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>What  said.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@FlyHighAirways plus you've added commercials to the experience... tacky.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>plus you've added commercials to the experience... tacky.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@FlyHighAirways I didn't today... Must mean I need to take another trip!</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>I didn't today... Must mean I need to take another trip!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp;amp; they have little recourse</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp;amp; they have little recourse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@FlyHighAirways and it's a really big bad thing about it</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>and it's a really big bad thing about it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>@FlyHighAirways seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>@FlyHighAirways yes, nearly every time I fly VX this “ear worm” won’t go away :)</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>yes, nearly every time I fly VX this “ear worm” won’t go away :)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@FlyHighAirways Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>@FlyHighAirways Well, I didn't…but NOW I DO! :-D</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>Well, I didn't…but NOW I DO! :-D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@FlyHighAirways it was amazing, and arrived an hour early. You're too good to me.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>it was amazing, and arrived an hour early. You're too good to me.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                         tweet  \\\n",
       "0                                                                                                         @FlyHighAirways What @dhepburn said.   \n",
       "1                                                                    @FlyHighAirways plus you've added commercials to the experience... tacky.   \n",
       "2                                                                     @FlyHighAirways I didn't today... Must mean I need to take another trip!   \n",
       "3              @FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp; they have little recourse   \n",
       "4                                                                                     @FlyHighAirways and it's a really big bad thing about it   \n",
       "5  @FlyHighAirways seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA   \n",
       "6                                                             @FlyHighAirways yes, nearly every time I fly VX this “ear worm” won’t go away :)   \n",
       "7                                @FlyHighAirways Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP   \n",
       "8                                                                                             @FlyHighAirways Well, I didn't…but NOW I DO! :-D   \n",
       "9                                                            @FlyHighAirways it was amazing, and arrived an hour early. You're too good to me.   \n",
       "\n",
       "                At  \\\n",
       "0  @FlyHighAirways   \n",
       "1  @FlyHighAirways   \n",
       "2  @FlyHighAirways   \n",
       "3  @FlyHighAirways   \n",
       "4  @FlyHighAirways   \n",
       "5  @FlyHighAirways   \n",
       "6  @FlyHighAirways   \n",
       "7  @FlyHighAirways   \n",
       "8  @FlyHighAirways   \n",
       "9  @FlyHighAirways   \n",
       "\n",
       "                                                                                                             tweet_preprocessed  \n",
       "0                                                                                                                   What  said.  \n",
       "1                                                                     plus you've added commercials to the experience... tacky.  \n",
       "2                                                                      I didn't today... Must mean I need to take another trip!  \n",
       "3               it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp; they have little recourse  \n",
       "4                                                                                      and it's a really big bad thing about it  \n",
       "5   seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA  \n",
       "6                                                              yes, nearly every time I fly VX this “ear worm” won’t go away :)  \n",
       "7                                 Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP  \n",
       "8                                                                                              Well, I didn't…but NOW I DO! :-D  \n",
       "9                                                             it was amazing, and arrived an hour early. You're too good to me.  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets['tweet_preprocessed'] = tweets['tweet'].apply(remove_handles)\n",
    "tweets.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sentiment(dataframe, column):\n",
    "    text_column = dataframe[column]\n",
    "    textblob_sentiment = text_column.apply(TextBlob)\n",
    "    sentiment_values = [ {'Polarity': v.sentiment.polarity, \n",
    "                          'Subjectivity': v.sentiment.subjectivity}\n",
    "                   for v in textblob_sentiment.values]\n",
    "    return pd.DataFrame(sentiment_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment_frame = get_sentiment(tweets, 'tweet_preprocessed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.390625</td>\n",
       "      <td>0.6875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.006250</td>\n",
       "      <td>0.3500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Polarity  Subjectivity\n",
       "0  0.000000        0.0000\n",
       "1  0.000000        0.0000\n",
       "2 -0.390625        0.6875\n",
       "3  0.006250        0.3500"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_frame.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.concat([tweets, sentiment_frame], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>At</th>\n",
       "      <th>tweet_preprocessed</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@FlyHighAirways What @dhepburn said.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>What  said.</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@FlyHighAirways plus you've added commercials to the experience... tacky.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>plus you've added commercials to the experience... tacky.</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@FlyHighAirways I didn't today... Must mean I need to take another trip!</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>I didn't today... Must mean I need to take another trip!</td>\n",
       "      <td>-0.390625</td>\n",
       "      <td>0.687500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp;amp; they have little recourse</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp;amp; they have little recourse</td>\n",
       "      <td>0.006250</td>\n",
       "      <td>0.350000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>@FlyHighAirways and it's a really big bad thing about it</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>and it's a really big bad thing about it</td>\n",
       "      <td>-0.350000</td>\n",
       "      <td>0.383333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>@FlyHighAirways seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA</td>\n",
       "      <td>-0.208333</td>\n",
       "      <td>0.633333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>@FlyHighAirways yes, nearly every time I fly VX this “ear worm” won’t go away :)</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>yes, nearly every time I fly VX this “ear worm” won’t go away :)</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>0.766667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@FlyHighAirways Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>@FlyHighAirways Well, I didn't…but NOW I DO! :-D</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>Well, I didn't…but NOW I DO! :-D</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@FlyHighAirways it was amazing, and arrived an hour early. You're too good to me.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>it was amazing, and arrived an hour early. You're too good to me.</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>0.600000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                         tweet  \\\n",
       "0                                                                                                         @FlyHighAirways What @dhepburn said.   \n",
       "1                                                                    @FlyHighAirways plus you've added commercials to the experience... tacky.   \n",
       "2                                                                     @FlyHighAirways I didn't today... Must mean I need to take another trip!   \n",
       "3              @FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp; they have little recourse   \n",
       "4                                                                                     @FlyHighAirways and it's a really big bad thing about it   \n",
       "5  @FlyHighAirways seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA   \n",
       "6                                                             @FlyHighAirways yes, nearly every time I fly VX this “ear worm” won’t go away :)   \n",
       "7                                @FlyHighAirways Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP   \n",
       "8                                                                                             @FlyHighAirways Well, I didn't…but NOW I DO! :-D   \n",
       "9                                                            @FlyHighAirways it was amazing, and arrived an hour early. You're too good to me.   \n",
       "\n",
       "                At  \\\n",
       "0  @FlyHighAirways   \n",
       "1  @FlyHighAirways   \n",
       "2  @FlyHighAirways   \n",
       "3  @FlyHighAirways   \n",
       "4  @FlyHighAirways   \n",
       "5  @FlyHighAirways   \n",
       "6  @FlyHighAirways   \n",
       "7  @FlyHighAirways   \n",
       "8  @FlyHighAirways   \n",
       "9  @FlyHighAirways   \n",
       "\n",
       "                                                                                                             tweet_preprocessed  \\\n",
       "0                                                                                                                   What  said.   \n",
       "1                                                                     plus you've added commercials to the experience... tacky.   \n",
       "2                                                                      I didn't today... Must mean I need to take another trip!   \n",
       "3               it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp; they have little recourse   \n",
       "4                                                                                      and it's a really big bad thing about it   \n",
       "5   seriously would pay $30 a flight for seats that didn't have this playing.\\r\\nit's really the only bad thing about flying VA   \n",
       "6                                                              yes, nearly every time I fly VX this “ear worm” won’t go away :)   \n",
       "7                                 Really missed a prime opportunity for Men Without Hats parody, there. https://t.co/mWpG7grEZP   \n",
       "8                                                                                              Well, I didn't…but NOW I DO! :-D   \n",
       "9                                                             it was amazing, and arrived an hour early. You're too good to me.   \n",
       "\n",
       "   Polarity  Subjectivity  \n",
       "0  0.000000      0.000000  \n",
       "1  0.000000      0.000000  \n",
       "2 -0.390625      0.687500  \n",
       "3  0.006250      0.350000  \n",
       "4 -0.350000      0.383333  \n",
       "5 -0.208333      0.633333  \n",
       "6  0.466667      0.766667  \n",
       "7  0.200000      0.200000  \n",
       "8  1.000000      1.000000  \n",
       "9  0.466667      0.600000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "positive_tweets = tweets[tweets.Polarity > 0.5]\n",
    "negative_tweets = tweets[tweets.Polarity <= - 0.5]\n",
    "neutral_tweets = tweets[ (tweets.Polarity > -0.1) & (tweets.Polarity < 0.1) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>At</th>\n",
       "      <th>tweet_preprocessed</th>\n",
       "      <th>Polarity</th>\n",
       "      <th>Subjectivity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@FlyHighAirways What @dhepburn said.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>What  said.</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>@FlyHighAirways plus you've added commercials to the experience... tacky.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>plus you've added commercials to the experience... tacky.</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp;amp; they have little recourse</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp;amp; they have little recourse</td>\n",
       "      <td>0.00625</td>\n",
       "      <td>0.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>@FlyHighAirways did you know that suicide is the second leading cause of death among teens 10-24</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>did you know that suicide is the second leading cause of death among teens 10-24</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>@FlyHighAirways SFO-PDX schedule is still MIA.</td>\n",
       "      <td>@FlyHighAirways</td>\n",
       "      <td>SFO-PDX schedule is still MIA.</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14626</th>\n",
       "      <td>@AmericanAir Flight 953 NYC-Buenos Aires has been delay since yesterday at 10PM. Is going to take off at 3.30PM now? Give us answers!</td>\n",
       "      <td>@AmericanAir</td>\n",
       "      <td>Flight 953 NYC-Buenos Aires has been delay since yesterday at 10PM. Is going to take off at 3.30PM now? Give us answers!</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14628</th>\n",
       "      <td>Thank you. “@AmericanAir: @jlhalldc Customer Relations will review your concerns and contact you back directly, John.”</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Thank you. “  Customer Relations will review your concerns and contact you back directly, John.”</td>\n",
       "      <td>0.05000</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14633</th>\n",
       "      <td>@AmericanAir my flight was Cancelled Flightled, leaving tomorrow morning. Auto rebooked for a Tuesday night flight but need to arrive Monday.</td>\n",
       "      <td>@AmericanAir</td>\n",
       "      <td>my flight was Cancelled Flightled, leaving tomorrow morning. Auto rebooked for a Tuesday night flight but need to arrive Monday.</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14635</th>\n",
       "      <td>@AmericanAir thank you we got on a different flight to Chicago.</td>\n",
       "      <td>@AmericanAir</td>\n",
       "      <td>thank you we got on a different flight to Chicago.</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14637</th>\n",
       "      <td>@AmericanAir Please bring American Airlines to #BlackBerry10</td>\n",
       "      <td>@AmericanAir</td>\n",
       "      <td>Please bring American Airlines to #BlackBerry10</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6730 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                               tweet  \\\n",
       "0                                                                                                               @FlyHighAirways What @dhepburn said.   \n",
       "1                                                                          @FlyHighAirways plus you've added commercials to the experience... tacky.   \n",
       "3                    @FlyHighAirways it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp; they have little recourse   \n",
       "10                                                  @FlyHighAirways did you know that suicide is the second leading cause of death among teens 10-24   \n",
       "15                                                                                                    @FlyHighAirways SFO-PDX schedule is still MIA.   \n",
       "...                                                                                                                                              ...   \n",
       "14626          @AmericanAir Flight 953 NYC-Buenos Aires has been delay since yesterday at 10PM. Is going to take off at 3.30PM now? Give us answers!   \n",
       "14628                         Thank you. “@AmericanAir: @jlhalldc Customer Relations will review your concerns and contact you back directly, John.”   \n",
       "14633  @AmericanAir my flight was Cancelled Flightled, leaving tomorrow morning. Auto rebooked for a Tuesday night flight but need to arrive Monday.   \n",
       "14635                                                                                @AmericanAir thank you we got on a different flight to Chicago.   \n",
       "14637                                                                                   @AmericanAir Please bring American Airlines to #BlackBerry10   \n",
       "\n",
       "                    At  \\\n",
       "0      @FlyHighAirways   \n",
       "1      @FlyHighAirways   \n",
       "3      @FlyHighAirways   \n",
       "10     @FlyHighAirways   \n",
       "15     @FlyHighAirways   \n",
       "...                ...   \n",
       "14626     @AmericanAir   \n",
       "14628              NaN   \n",
       "14633     @AmericanAir   \n",
       "14635     @AmericanAir   \n",
       "14637     @AmericanAir   \n",
       "\n",
       "                                                                                                                      tweet_preprocessed  \\\n",
       "0                                                                                                                            What  said.   \n",
       "1                                                                              plus you've added commercials to the experience... tacky.   \n",
       "3                        it's really aggressive to blast obnoxious \"entertainment\" in your guests' faces &amp; they have little recourse   \n",
       "10                                                      did you know that suicide is the second leading cause of death among teens 10-24   \n",
       "15                                                                                                        SFO-PDX schedule is still MIA.   \n",
       "...                                                                                                                                  ...   \n",
       "14626           Flight 953 NYC-Buenos Aires has been delay since yesterday at 10PM. Is going to take off at 3.30PM now? Give us answers!   \n",
       "14628                                   Thank you. “  Customer Relations will review your concerns and contact you back directly, John.”   \n",
       "14633   my flight was Cancelled Flightled, leaving tomorrow morning. Auto rebooked for a Tuesday night flight but need to arrive Monday.   \n",
       "14635                                                                                 thank you we got on a different flight to Chicago.   \n",
       "14637                                                                                    Please bring American Airlines to #BlackBerry10   \n",
       "\n",
       "       Polarity  Subjectivity  \n",
       "0       0.00000          0.00  \n",
       "1       0.00000          0.00  \n",
       "3       0.00625          0.35  \n",
       "10      0.00000          0.00  \n",
       "15      0.00000          0.00  \n",
       "...         ...           ...  \n",
       "14626   0.00000          0.00  \n",
       "14628   0.05000          0.20  \n",
       "14633   0.00000          0.00  \n",
       "14635   0.00000          0.60  \n",
       "14637   0.00000          0.00  \n",
       "\n",
       "[6730 rows x 5 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "positive_tweets.head(15)\n",
    "negative_tweets.head(15)\n",
    "neutral_tweets"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}