Sentiment-Bayes.ipynb

{
 "metadata": {
  "name": "Sentiment-Bayes"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Approach:\n",
      "\n",
      "1) identify high frequency hashtags and use of emoticons\n",
      "2) Determine sets indicatig positive and negative sentiment\n",
      "    POS(#) is hashtag\n",
      "    POS(E) is emoticons\n",
      "3) Create training set based on high sentiment terms\n",
      "4) Train hayes filter\n",
      "    a) try using unigrams only, filter out sentiment indicators\n",
      "    b) do same using bigrams  << not done yet\n",
      "5) evaluate predictors using tweets w/o sentiment indicators.\n",
      "\n",
      "to do:  look at Stanford paper"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%pylab inline\n",
      "plt.rc('figure', figsize=(8, 8))\n",
      "\n",
      "import cPickle as pickle\n",
      "\n",
      "import sys\n",
      "sys.path.append('/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/')\n",
      "sys.path.append('/Users/doug/SW_Dev/NLTK_Experiments/')\n",
      "\n",
      "import pprint\n",
      "import collections\n",
      "import operator\n",
      "\n",
      "import my_feature_ex as fx\n",
      "import my_word_cloud as wcloud"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Welcome to pylab, a matplotlib-based Python environment [backend: module://IPython.zmq.pylab.backend_inline].\n",
        "For more information, type 'help(pylab)'.\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "skipped:  111010100010\t\t212\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import nltk\n",
      "import json\n",
      "sys.path.append('/Users/doug/SW_Dev/ark-tweet-nlp-0.3.2')\n",
      "import CMUTweetTagger\n",
      "#print CMUTweetTagger.runtagger_parse(['example tweet 1', '@foo example tweet 2'])\n",
      "\n",
      "RUN_TAGGER_CMD = \"java -XX:ParallelGCThreads=2 -Xmx500m -jar /Users/doug/SW_Dev/ark-tweet-nlp-0.3.2/ark-tweet-nlp-0.3.2.jar\"\n",
      "RUN_TAGGER_CMD_PTB = \"java -XX:ParallelGCThreads=2 -Xmx500m -jar /Users/doug/SW_Dev/ark-tweet-nlp-0.3.2/ark-tweet-nlp-0.3.2.jar --model /Users/doug/SW_Dev/ark-tweet-nlp-0.3.2/model.ritter_ptb_alldata_fixed.20130723.txt\"\n",
      "print CMUTweetTagger.runtagger_parse(['example tweet 1', 'example tweet 2'], run_tagger_cmd=RUN_TAGGER_CMD)\n",
      "print CMUTweetTagger.runtagger_parse(['example tweet 1', 'example tweet 2'], run_tagger_cmd=RUN_TAGGER_CMD_PTB)\n",
      "\n",
      "def annotate_pos(tweets, ptb=False):\n",
      "    if ptb:\n",
      "        tagger_cmd = RUN_TAGGER_CMD_PTB\n",
      "    else:\n",
      "        tagger_cmd = RUN_TAGGER_CMD\n",
      "    ids = []\n",
      "    texts = []\n",
      "    for key, value in tweets.items():\n",
      "        ids.append(key)\n",
      "        texts.append(json.dumps({'text':value['text']}))\n",
      "    pos = CMUTweetTagger.runtagger_parse(texts, run_tagger_cmd=tagger_cmd)\n",
      "    \n",
      "    if len(ids) != len(pos):\n",
      "        raise Exception(\"Error: Tweet Tagger returned incorrect results\") \n",
      "\n",
      "    for i in range(0, len(ids)):\n",
      "        tweets[ids[i]]['pos'] = pos[i]\n",
      "        tweets[ids[i]]['tokens'] = [tag[0] for tag in pos[i]]\n",
      "\n",
      "    print pos[0]\n",
      "    print [tag[0] for tag in pos[0]]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[('example', 'N', 0.979), ('tweet', 'V', 0.7763), ('1', '$', 0.9916)], [('example', 'N', 0.979), ('tweet', 'V', 0.7713), ('2', '$', 0.5832)]]\n",
        "[[('example', 'NN', 0.5422), ('tweet', 'NN', 0.6582), ('1', 'CD', 0.8683)], [('example', 'NN', 0.5422), ('tweet', 'NN', 0.6071), ('2', 'TO', 0.3739)]]"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def load_tweet_corpus(fname):\n",
      "    tweets = dict()\n",
      "    fd = open(fname)\n",
      "    for json_text in fd:\n",
      "        tweet = json.loads(json_text)['tweet']\n",
      "        tweets[tweet['id_str']] = tweet\n",
      "    print 'loaded'\n",
      "    return tweets\n",
      "\n",
      "def load_UCLA_tweet_corpus():\n",
      "    fname = '/Users/doug/Desktop/NLP/sentiment/UCLA.json'\n",
      "    return load_tweet_corpus(fname)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "UCLA_tweets = load_UCLA_tweet_corpus() \n",
      "\n",
      "annotate_pos(UCLA_tweets)\n",
      "#annotate_pos(UCLA_tweets, ptb=True)\n",
      "\n",
      "#subset_tweets = [t['text'] for t in UCLA_tweets.values()[:10]]\n",
      "#pprint.pprint(CMUTweetTagger.runtagger_parse(subset_tweets))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "loaded\n",
        "[('Should', 'V', 0.9952), ('I', 'O', 0.9994), ('worry', 'V', 0.9988), (',', ',', 0.9982), ('or', '&', 0.9936), ('take', 'V', 0.9366), ('comfort', 'N', 0.9568), ('?', ',', 0.9972), ('@UCLAnewsroom10m', '@', 0.9761), ('Eating', 'V', 0.9401), ('probiotics', 'N', 0.9535), ('affects', 'V', 0.9974), ('brain', 'N', 0.9788), ('function', 'N', 0.9814), (',', ',', 0.9967), ('UCLA', '^', 0.9757), ('study', 'N', 0.8871), ('shows', 'V', 0.7623), ('.', ',', 0.998), ('http://t.co/eskRkWb4Bl', 'U', 0.9984)]"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "['Should', 'I', 'worry', ',', 'or', 'take', 'comfort', '?', '@UCLAnewsroom10m', 'Eating', 'probiotics', 'affects', 'brain', 'function', ',', 'UCLA', 'study', 'shows', '.', 'http://t.co/eskRkWb4Bl']\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#import cPickle as pickle\n",
      "\n",
      "#poutput = open('ucla_tweets.pkl', 'wb')\n",
      "#pickle.dump(UCLA_tweets, poutput, -1)\n",
      "#poutput.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# load previously tokenized/clasified  tweet Corpus\n",
      "#pinput = open('ucla_tweets.pkl', 'rb')\n",
      "#UCLA_tweets = pickle.load(pinput)\n",
      "#pinput.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "MAX_TWEETS = 100000   #subset Corpus for now to improve Clustering run time\n",
      "tweet_set = [{'text':t['text'], 'pos':t['pos'], 'raw_tokens':t['tokens']} for t in UCLA_tweets.values()[0:MAX_TWEETS]]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def compute_word_counts(tweet_set):\n",
      "    wc = collections.defaultdict(int)\n",
      "    for tweet in tweet_set:\n",
      "        for tok in tweet['tokens']:\n",
      "            wc[tok] += 1\n",
      "    sorted_wc = sorted(wc.iteritems(), key=operator.itemgetter(1), reverse=True)\n",
      "    sorted_wc = [val for val in sorted_wc if val[1]>1]   #prune singleton values\n",
      "    #print len(sorted_wc)\n",
      "    words = [word for word, count in sorted_wc]\n",
      "    counts = [count for word, count in sorted_wc]\n",
      "    return words, counts\n",
      "    \n",
      "def display_tweet_wordcloud(tweet_set):\n",
      "    words, counts = compute_word_counts(tweet_set)\n",
      "    words2 =  np.array(words[:200], np.dtype('string'))\n",
      "    counts2 =  np.array(counts[:200], np.int32)\n",
      "    wcloud.display_wordcloud(words2,counts2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def display_word_counts(tweet_set, exclude=[]):\n",
      "    wc = collections.defaultdict(int)\n",
      "    for tweet in tweet_set:\n",
      "        for tok in tweet['tokens']:\n",
      "            wc[tok] += 1\n",
      "    sorted_wc = sorted(wc.iteritems(), key=operator.itemgetter(1), reverse=True)\n",
      "    sorted_wc = [val for val in sorted_wc if val[1]>1]   #prune singleton values\n",
      "    for word, count in sorted_wc:\n",
      "        if word not in exclude:\n",
      "            print count, ' : ', word"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#create word cloud based on emoticons"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for tweet in tweet_set:\n",
      "    tweet['tokens'] = [val for (val, pos, x) in tweet['pos'] if pos == 'E']\n",
      "    if tweet['tokens']:\n",
      "        #print tweet['tokens'], ':', tweet['text']\n",
      "        pass"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#display_tweet_wordcloud(tweet_set)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#display_word_counts(tweet_set)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "emot_positive = [':)', ':D', '<3', ':-)', ':))', '(-:', '\\xe2\\x9d\\xa4','\\xe2\\x98\\xba', '\\xf0\\x9f\\x98\\x8a', ':)))','(:',\n",
      "            ':o)',  ':]',  ':3',  ':c)',  ':>',  '=]',  '8)',  '=)',  ':}',  ':^)',  ':\u3063)', ':-))', ':D',\n",
      "            ':-D', ':D',  '8-D',  '8D',  'x-D',  'xD',  'X-D',  'XD',  '=-D',  '=D',  '=-3',  '=3',  'B^D',\n",
      "            \":'-)\", \":')\", 'QQ','\\xef\\xb8\\x8f', '\\xe2\\x99\\xa5', '\\xe2\\x99\\xa5\\xe2\\x99\\xa5\\xe2\\x99\\xa5','\\xe2\\x99\\xa5\\xe2\\x99\\xa5\\xe2\\x99\\xa5\\xe2\\x99\\xa5',\n",
      "            '^.^', '\\xf0\\x9f\\x98\\x8d', '\\xf0\\x9f\\x91\\x8c', '\\\\(^_^)', '<333', '<33','\ud83d\udc4d', '\ud83d\udc9b', '\ud83d\udc9a', '\u2764\u2764\u2764\u2764', '\u263a', '\ud83d\udc95',\n",
      "            ]\n",
      "emot_negative = [':(', \":'(\", '>:[',  ':-(',  ':-c',  ':c',  ':-<',  ':\u3063C',  ':<',  ':-[',  ':[',  ':{',\n",
      "            ':-||', ':@', '>:(', \":'-(\", \":'(\", 'D:<', 'D:', 'D8', 'D;', 'D=', 'DX', 'v.v', \"D-':\",\n",
      "            '>:\\\\', '>:/', ':-/', ':-.', ':/', ':\\\\', '=/', '=\\\\', ':L', '=L', ':S', '>.<',\n",
      "            ':-###..', ':###..', '\\xf0\\x9f\\x98\\xaa', ';o', '-.-',\n",
      "           ]\n",
      "emot_ignore = ['D:', 'x', ';-)', ';)', '*-)', '*)', ';-]', ';]', ';D', ';^)', ':-,'\n",
      "          '>:P', ':-P', ':P', 'X-P', 'x-p', 'xp', 'XP', ':-p', ':p', '=p', ':-\u00de', ':\u00de', ':\u00fe', ':-\u00fe', ':-b', ':b',\n",
      "          'X', ':', '($)', ')', ']', ':o', '-_-', '>', '\u26be', '\ud83d\udc3b', '\ud83c\udf93', '(', '\ud83d\ude4f', '\ud83d\udc81',\n",
      "         ]\n",
      "exclude = emot_positive + emot_negative + emot_ignore\n",
      "#display_word_counts(tweet_set, exclude=exclude)\n",
      "print '------------------------------------------'\n",
      "for tweet in tweet_set:\n",
      "    tweet['tokens'] = [val for (val, pos, x) in tweet['pos'] if pos == 'E' and val not in exclude]\n",
      "    if tweet['tokens']:\n",
      "        #print tweet['tokens'], ':', tweet['text']\n",
      "        pass"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "------------------------------------------\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#create word cloud based on hashtags"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for tweet in tweet_set:\n",
      "    tweet['tokens'] = [val for (val, pos, x) in tweet['pos'] if pos == '#']\n",
      "#display_tweet_wordcloud(tweet_set)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "hash_positive = ['#Love', '#beautiful', '#win','#westcoastlove','#Smile','#DreamCollege',\n",
      "                 '#dreamschool', '#love', '#cute', '#Salute', '#cantwait', '#awesome', '#HIT', '#lucky',\n",
      "                 '#inspiring', '#sohappy', '#inspiration', '#happy', '#Proud', '#yessssss', '#BEAUTIFUL', \n",
      "                 '#Blessed', '#WOW',  # ignore  '#WeLoveLA', '#Hot', ]\n",
      "                ]\n",
      "hash_negative = ['#fail', '#scared', '#no', '#wtf', '#fail', '#shady', \n",
      "                 '#pathetic', '#fuckit', '#scary', '#corruption', '#gross', '#repulsive', '#disgusting', '#greed',\n",
      "                ]\n",
      "\n",
      "exclude = ['#WeLoveLA', '#Hot', ]\n",
      "\n",
      "hash_positive = [val.lower() for val in hash_positive]\n",
      "hash_negative = [val.lower() for val in hash_negative]\n",
      "exclude = [val.lower() for val in exclude]\n",
      "\n",
      "#display_word_counts(tweet_set)\n",
      "#display_word_counts(tweet_set, exclude=exclude)\n",
      "print '------------------------------------------'\n",
      "for tweet in tweet_set:\n",
      "    tweet['tokens'] = [val for (val, pos, x) in tweet['pos'] if val.lower() in hash_positive and val.lower() not in exclude]\n",
      "    if tweet['tokens']:\n",
      "        #print tweet['tokens'], ':', tweet['text']\n",
      "        pass"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "------------------------------------------\n"
       ]
      }
     ],
     "prompt_number": 58
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def dedupe_string(s):  #truncates replications of 3 for more of the same letter to max of two instances\n",
      "    result = [a for (a,b,c) in zip(s, s[1:], s[2:]) if a != b and a != c]\n",
      "    #result = result + s[-2:]\n",
      "    return ''.join(result)+ s[-2:]\n",
      "\n",
      "def dedupe_tokens(tokens, dedupe=True):\n",
      "    if dedupe:\n",
      "        return [dedupe_string(tok) for tok in tokens]\n",
      "    else:\n",
      "        return tokens\n",
      "    \n",
      "#dedupe_tokens([\"soooo\", \"whaaaaat\"])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 69,
       "text": [
        "['soo', 'what']"
       ]
      }
     ],
     "prompt_number": 69
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#to do:\n",
      "#  change lematize versions to call dedupe_string() prior to lemmatize rather than dedupe_tokens() at end\n",
      "\n",
      "#Variations of my_feature_ex.py that return vectors\n",
      "from nltk.stem import WordNetLemmatizer\n",
      "wnl = WordNetLemmatizer()\n",
      "def get_tokens(tweet, filter='default', exclude=[], dedupe=True):\n",
      "    t = [t.lower() for t,p,f in tweet['pos'] if fx.filter_token(t,p,f,filter=filter, exclude=exclude)]\n",
      "    return dedupe_tokens(t, dedupe=dedupe)\n",
      "\n",
      "def get_lemmatize_tokens(tweet, filter='default', exclude=[], dedupe=True):\n",
      "    t = [wnl.lemmatize(t.lower()) for t,p,f in tweet['pos'] if fx.filter_token(t,p,f,filter=filter, exclude=exclude)]\n",
      "    return dedupe_tokens(t, dedupe=dedupe)\n",
      "    \n",
      "def get_group_tokens(tweet, filter='default', exclude=[], dedupe=True):\n",
      "    t = [cmu_tweet_word_clusters.cmu_classify(t.lower()) for t,p,f in tweet['pos'] \n",
      "                                                         if fx.filter_token(t,p,f,filter=filter, exclude=exclude)]\n",
      "    return dedupe_tokens(t, dedupe=dedupe)\n",
      "    \n",
      "def get_bigrams(tweet, filter='default', exclude=[], dedupe=True):\n",
      "    bigrams = zip(tweet['pos'][0:-1], tweet['pos'][1:])\n",
      "    t = ['{}_{}'.format(pos1[0].lower(), pos2[0].lower()) for pos1, pos2 in bigrams \n",
      "                                                          if fx.filter_token(*pos1,filter=filter,exclude=exclude) \n",
      "                                                          and fx.filter_token(*pos2,filter=filter,exclude=exclude)]\n",
      "    return dedupe_tokens(t, dedupe=dedupe)\n",
      "\n",
      "def get_lemmatize_bigrams(tweet, filter='default', exclude=[], dedupe=True):\n",
      "    bigrams = zip(tweet['pos'][0:-1], tweet['pos'][1:])\n",
      "    t = ['{}_{}'.format(wnl.lemmatize(pos1[0].lower()), wnl.lemmatize(pos2[0].lower())) \n",
      "         for pos1, pos2 in bigrams if  fx.filter_token(*pos1,filter=filter,exclude=exclude) \n",
      "                                   and fx.filter_token(*pos2,filter=filter,exclude=exclude)]\n",
      "    return dedupe_tokens(t, dedupe=dedupe)\n",
      "\n",
      "#ignore for now\n",
      "def get_group_bigrams(tweet, filter='default', exclude=[], dedupe=True):\n",
      "    bigrams = zip(tweet['pos'][0:-1], tweet['pos'][1:])\n",
      "    t = ['{}_{}'.format(cmu_tweet_word_clusters.cmu_classify(pos1[0].lower()), cmu_tweet_word_clusters.cmu_classify(pos2[0].lower())) \n",
      "         for pos1, pos2 in bigrams if fx.filter_token(*pos1,filter=filter,exclude=exclude) \n",
      "                                   and fx.filter_token(*pos2,filter=filter,exclude=exclude)]\n",
      "    return dedupe_tokens(t, dedupe=dedupe)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 70
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.vector import Document, NB, KNN, SVM\n",
      "\n",
      "def train_classifier(classifier, tweet_set, positive, negative, extract=get_tokens):\n",
      "    sentiment = positive.union(negative)\n",
      "    for tweet in tweet_set:\n",
      "        tokens = set([val.lower() for val in tweet['raw_tokens']])\n",
      "        if tokens.intersection(sentiment):\n",
      "            #print tweet['text']\n",
      "            polarity = 0\n",
      "            if tokens.intersection(positive): polarity += 1\n",
      "            if tokens.intersection(negative): polarity -= 1\n",
      "            if polarity != 0:\n",
      "                #print \"    \",tokens.difference(sentiment)\n",
      "                t = extract(tweet, filter='default', exclude=sentiment)\n",
      "                v = Document(t, stopwords=False, type=int(polarity))\n",
      "                #v = Document(list(tokens.difference(sentiment)), stopwords=False, type=int(polarity))\n",
      "                classifier.train(v)\n",
      "\n",
      "def classify_tweets(classifier, tweet_set, positive, negative, extract=get_tokens):\n",
      "    sentiment = positive.union(negative)\n",
      "    for tweet in tweet_set:\n",
      "        print '[',\n",
      "        tokens = set([val.lower() for val in tweet['raw_tokens']])\n",
      "        if tokens.intersection(sentiment):\n",
      "            #print '[',\n",
      "            polarity = 0\n",
      "            if tokens.intersection(positive): polarity += 1\n",
      "            if tokens.intersection(negative): polarity -= 1\n",
      "            print polarity, ',',\n",
      "        \n",
      "        t = extract(tweet, filter='default', exclude=sentiment)\n",
      "        v = Document(t, stopwords=False)\n",
      "        #v = Document(list(tokens), stopwords=False)\n",
      "        if type(classifier) == NB or type(classifier) == KNN: \n",
      "            #if True : \n",
      "            prediction = classifier.classify(v, discrete=False)\n",
      "            #prediction = defaultdict(int, prediction)\n",
      "            #print prediction\n",
      "            p = prediction[1] - prediction[-1]\n",
      "            print \"%.1f\" % p, '] ', tweet['text']\n",
      "            #print prediction[1]\n",
      "            #print prediction[-1]\n",
      "            #print prediction\n",
      "        else: \n",
      "            prediction = classifier.classify(v)\n",
      "            print prediction, '] ', tweet['text'] "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 106
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#positive = set(emot_positive + hash_positive)\n",
      "#negative = set(emot_negative + hash_negative)\n",
      "positive = set(emot_positive)\n",
      "negative = set(emot_negative)\n",
      "\n",
      "# Inputs to program\n",
      "#  tweet[tokens] -- filtered set of tokens\n",
      "#  tweet[raw_tokens]\n",
      "classifier = NB()\n",
      "classifier = KNN()\n",
      "#classifier = SVM()\n",
      "\n",
      "train_classifier(classifier, tweet_set, positive, negative, extract=get_lemmatize_bigrams)\n",
      "classify_tweets(classifier,tweet_set[:200], positive, negative, extract=get_lemmatize_bigrams)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[ 1.0 ]  Should I worry, or take comfort? @UCLAnewsroom10m Eating probiotics affects brain function, UCLA study shows. http://t.co/eskRkWb4Bl\n",
        "[ 0.0 ]  I want a UCLA sweater so bad!!!!\n",
        "[ 0.0 ]  @CateEsser Thanks Cate! But, who are the Bruins? UCLA?? And Which Sport were you watching anyways??? LOL's!\n",
        "[ 0.0 ]  Race Day this Saturday at UCLA. See you there :D\n",
        "[ -1.0 ]  Free concert at ucla tonight! #enemyterritory\n",
        "[ 0.0 ]  At UCLA gettin this track work in\n",
        "[ 0.5 ]  Wannah Bail (corrected) is now eligible and expected to visit UCLA tomorrow. Source says that UCLA and Miami r top contenders to land him.\n",
        "[ 1 , 1.0 ]  Can't wait till I'm a jr. So I can go to there opening ! \u2764 #ucla\n",
        "[ 0.0 ]  @D_Squirtle we really did ! Lol I wish all Moore league schools can do workshops w/ UCLA pier , we'd be turnt w/ JTown!\n",
        "[ -1.0 ]  At  my boo boo bears graduation... Crazy how time flies... Just yesterday we were bringing her home from ucla... #proudbrother\n",
        "[ 0.0 ]  UCLA construction mishap http://t.co/aoIwCx3fiP\n",
        "[ 0.0 ]  I swear the day in accepted into either fsu or ucla, ill KNOW this hard work payed off\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.0 ]  Tonights off-site catering. Benefiting UCLA Jonsson Cancer Center Foundation at Le Merigot Hotel http://t.co/ue3s9no4j9\n",
        "[ 0.0 ]  Wow...  http://t.co/KSdjcPFfWT\n",
        "[ 0.9 ]  \"@comptoncarlson: Is anyone going to see Grant Watson pitch at UCLA today? I really need a ride.\" get on the Get bus\n",
        "[ 1.0 ]  OT - 5 Star DT Eddie Vanderdoes Not Going to ND: He will take his talents to UCLA\u2026 http://t.co/eTB6pUu4wk http://t.co/FmRRcKc9x4 #MGoBoard\n",
        "[ -1.0 ]  the fact that James Franco is a teacher at UCLA makes me more excited to be there\n",
        "[ 0.0 ]  Behind the scenes of the UCLA's Men's Basketball locker room \ud83d\ude4a\ud83c\udfc0 #teehee #ilovemyjob #sas #baloncesto\n",
        "[ 0.0 ]  UCLA!!! UCLA is doing a great job! #rugbg7s\n",
        "[ 0.0 ]  @OKBJGM @MuseZack this particular one I read at UCLA, but I instantly thought of you ;)\n",
        "[ 0.0 ]  Irish DT recruit Vanderdoes going to UCLA http://t.co/oQLOdAbB7w\n",
        "[ 0.0 ]  UCLA's rugby captain is 5'4, THERE IS HOPE.\n",
        "[ 0.0 ]  Is FAMU worth relocating? is UCLA worth relocating?\n",
        "[ 0.0 ]  Are children who take Ritalin for ADHD at greater risk of future drug abuse? http://t.co/vs1Wk0y3Zk\n",
        "[ -1.0 ]  @vanessabell_ hey I know the kid in the UCLA shirt.\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.0 ]  @D1JayyBerk Thank you and UCLA sir\n",
        "[ 0.0 ]  Bryant will bat second this inning. no hits today and just 2 INF singles all tournament. #UCLA 6, #USD 0 (T9).\n",
        "[ 0.0 ]  @KillaBeeG_323 hopefully USC or UCLA\n",
        "[ 0.0 ]  Everyone follow and congratulate @Simply_brookeee for banking UCLA. We like us some smart chicks\n",
        "[ 0.0 ]  UCLA researchers may have uncovered a new treatment for Cushing's disease. http://t.co/7LG1JTIMCG #Cushingsdisease #treatment\n",
        "[ 0.0 ]  UCLA Anderson Forecast paints dismal picture of economic recovery http://t.co/E1VvGZJYRm\n",
        "[ 1.0 ]  Baseball Regionals: Fri May 31: UCLA 5, San Diego State 2 Oregon State 5, Texas San Antonio 4 Mississippi State... http://t.co/WOpqyaPjaA\n",
        "[ 0.0 ]  UCLA this summer!!! Is this for real? #ohmyohmyohmy\n",
        "[ 0.0 ]  @MattJonesNCAA Also proof that it actually is a gamble not to go to Kentucky as a top prospect. You can't go just anywhere, not even UCLA\n",
        "[ 0.0 ]  USA 7'S RUGBY -- UCLA 12 - NAVY 7 WOOHOO #ItTakesLeatherBalls http://t.co/GOZ6XU59Px #GetGlue @NBC\n",
        "[ -1 , 0.0 ]  One more year and then ucla :/\n",
        "[ 0.0 ]  \"Sometimes you go to bed thinking about 3-4 issues, and wake up with 1 more.\" - Kofi Annan #UCLA #UN\n",
        "[ 0.7 ]  Five-star shooting guard Rashad Vaughn on his Westwood unofficial --&gt; RT @ShowtimeMr UCLA campus is beautiful\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "-0.2 ]  If I'm accepted to UCLA, UCSD or Berkeley for grad school, I'm gonna be the happiest girl in the world.\n",
        "[ 1.0 ]  @SithLordFlo You just have to look out for the bracelet Flo. Lets go to UCLA maybe we will find some there.\n",
        "[ 0.0 ]  @chhart12193 I was amazed by the guy who jumped over the tiny ucla player!\n",
        "[ 0.3 ]  UCLA Bruins Flags http://t.co/U0PqPrIgJh\n",
        "[ 0.0 ]  Couple of shots from today's shoot with @LydiaEmillen for Ucla / Womenswear Buyer Mag. #shootingbabesisaneasyjob http://t.co/MbYOqJtpNL\n",
        "[ -1.0 ]  \u201c@_karinavee: I love her 1000x more than Mrs. P #sorrynotsorry &amp; that UCLA shirt is just a plus \ue420 http://t.co/JCJN0hiXh1\u201d\n",
        "[ -0.1 ]  Justine Bateman, 47, Goes Back to School as a Freshman at UCLA! http://t.co/Zik4shr8p2\n",
        "[ 0.0 ]  It's time for baseball: #UCLA and @USDToreros in the championship game of the LA regional. http://t.co/X43y0vfh1b\n",
        "[ 1.0 ]  Changing gut bacteria through diet affects brain function, UCLA study shows http://t.co/GqdTn7hU0s #neuroscience\n",
        "[ 0.0 ]  Congrats to @uclamsoccer's Carlos Bocanegra @BocaBoca3 for his induction into the #UCLA Athletics Hall of Fame http://t.co/JXvf0IMm5E\n",
        "[ 0.0 ]  Assign girls everywhere, UCLA\n",
        "[ 0.3 ]  Rugby 7s = Fun. Check out this sick hurdle in the Cal vs UCLA game.  http://t.co/h1oKDTbOcw http://t.co/Gvl06WMX9g\n",
        "[ 0.0 ]  Desert Mountain's Kyle Allen chose Texas A&amp;M... Our guess was UCLA. Congrats to the Arizona product.\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "-1.0 ]  @Beerawx3 ima miss yu bro\ud83d\ude22... Lol we playin UCLA thou\n",
        "[ 0.0 ]  Trying to convince my mom and dad to take me to UCLA on Friday \ud83d\ude2c\ud83d\ude0f\ud83d\ude0f #prettyplease\n",
        "[ 0.0 ]  #69FactsAboutMe I want to go study performing arts &amp; phycology @UCLA when I turn 18\n",
        "[ 0.0 ]  I'm on my ballin' each and every day\n",
        "Asian girls everywhere, UCLA\n",
        "[ 0.0 ]  I just wanna do the UCLA Undie Run \ud83d\ude29.\n",
        "[ 0.7 ]  Next class... @ UCLA Anderson School of Management http://t.co/g4nDw5agrd\n",
        "[ 0.0 ]  This is a BRILLIANT thread on the UCLA Confessions Facebook page: https://t.co/eVNRIkOEy8\n",
        "[ 0.0 ]  @Jabra_US I want your headphones sooooo bad!!!!!! #Jabra #ucla http://t.co/bAwdeJDCDP\n",
        "[ 0.0 ]  Irish signee Vanderdoes switches to UCLA instead\n",
        "http://t.co/1OPQ58VKx8\n",
        "[ 0.0 ]  #UCLA's newest commit OT Dominick Jackson is the No. 2-ranked JUCO in @247Composite http://t.co/RLNHfIpAAE\n",
        "[ 0.0 ]  UCLA Anderson Forecast paints dismal picture of economic recovery http://t.co/E7c3POeID1 #Depressing\n",
        "[ -0.3 ]  there's just no way in hell im going back to ucla for two weeks. im in my summer zone and im not tryna leave.\n",
        "[ 0.7 ]  UCLA campus? No, Paris in June #niceweather  http://t.co/DdKnfib4PC\n",
        "[ 1.0 ]  Eddie Vanderdoes, one-time Notre Dame signee, headed to UCLA http://t.co/oPzWMLjVmQ\n",
        "[ 1.0 ]  I must go to either USC or UCLA for grad school. It's a must #dreamchaser #biggoal\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.3 ]  #UCLA: Eddie Vanderdoes Will Be Well Worth the Wait for UCLA Bruins http://t.co/hOmTVE9mPl\n",
        "[ 0.0 ]  Asian girls everywhere, UCLA\n",
        "[ 0.0 ]  UCLA researchers identify abnormal brain networks in Fragile X syndrome http://t.co/P3GIy716rz\n",
        "[ 0.0 ]  #WeLoveLA UCLA at Stanford Saturday Baseball Thread: Bruins Try To Even Things Up - Bruins Nation http://t.co/7EpvsnNH5Z #SportsRoadhouse\n",
        "[ 0.0 ]  #UCLA The 1st step to building credit history is here. More Info and FAQ's at http://t.co/Fe9XZj0H2J\n",
        "[ 0.0 ]  Less than 2 weeks and my first year at UCLA will be over. This is such a surreal feeling.\n",
        "[ 0.0 ]  UCLA snagged a 5-star ranked defensive lineman from Notre Dame! \ud83d\ude28\ud83d\ude0d\n",
        "[ 0.0 ]  @JheneAiko Talks \"Comfort Inn Ending,\" Performs at @UCLA Jam http://t.co/wuQIBt3bhN @JazzReggaeFest\n",
        "[ 0.0 ]  Sophomore David Berg has been named as one of five finalists for the @NCBWA Stopper of the Year Award. http://t.co/qhxJccE7La\n",
        "[ 1.0 ]  SOCAL FRIENDS, I'm in need of several UCLA Band uniforms!! Willing to rent &amp; pay shipping, will be returned DRY CLEANED. Contact me ASAP!!\n",
        "[ -1.0 ]  Former UCLA star's dad under house arrest: A federal magistrate on Friday ordered the father of former UCLA ba... http://t.co/T2YsT1RGT1\n",
        "[ 0.0 ]  @Newswise- #Gallup and #UCLA poll finds that healthy lifestyle means fewer #memory complaints/\n",
        "[ 1.0 ]  UCLA bound\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1.0 ]  @WillyjWill transfer major into UCLA after a year at either Santa Monica or pierce . #3rdLegacy \ud83d\ude0e and I'm Native American which helps alot\n",
        "[ 0.0 ]  Ice Miller report says game bewteen UCLA and #arizonawildcats was officiated fairly and Ed Rush was joking with bounty offer.on Miller\n",
        "[ 0.9 ]  Really wish I could be at UCLA today to hear @footage talk today &amp; see the movie. It's all local. This portfolio kinda scrimmaged everything\n",
        "[ 0.0 ]  I want to go to UCLA bad!\n",
        "[ 0.0 ]  #chillin at the #fari #apparel #booth #ucla #jazz #reggae #festival #losangeles #california #stjohn\u2026 http://t.co/LOqzEzdmVu\n",
        "[ 0.0 ]  Yes, my librarian buddy.. UCLA is around the corner. We have to hang out before you leave. And im trapped in this house. Saaaave me!!!!\n",
        "[ 0.0 ]  Watson's error on a pickoff put a runner on third, but no damage to UCLA as he ends the inning soon after. Only needed 84 pitches in 7 IP.\n",
        "[ 0.0 ]  \u201c@cecejeferson7: #USC #UCLA and #Stanford June 23-26 \n",
        "#SunsetStrip #Cali\u201d Cece Jefferson\n",
        "[ 0.0 ]  Yes!!! RT @UCLAHealth: \"...UCLA Surgeons Live Vine Brain Surgery http://t.co/RKfRZoURAn via @mashable\u201d\n",
        "[ 0.0 ]  UCLA Lightning #EnemyBracelets #bracelets  #scene #UCLA http://t.co/UW7Ww2RB3C\n",
        "[ 0.0 ]  Time to write a movie #study #ucla #finals #grindin #la @ Casa del Diamente http://t.co/gBRuyjV4rw\n",
        "[ 0.0 ]  I'M JUSTINE BATEMAN. I'M 47. I'M A COLLEGE FRESHMAN AT UCLA. http://t.co/zd1M73hB0e\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.0 ]  #OKState included in 4* QB Kyle Allen's top 5 along with Notre Dame, #UCLA, A&amp;M, and Ohio State. #GoPokes\n",
        "[ 0.0 ]  Marble Falls WR Garrett Gray picked up UCLA and Oklahoma State offers.\n",
        "[ 0.3 ]  Eddie Vanderdoes Officially Leaving Notre Dame for UCLA Bruins http://t.co/1F4klt0nhy\n",
        "[ 0.0 ]  #UCLA IS ON THE RISE. THAT SCHOOL IS TRYING TO GET BETTER, I'M TELLING YA.\n",
        "[ 0.0 ]  #UCLA UCLAAthletics: The women's @UCLA_Rowing V8+ crew finishes 2nd in first-round NCAA... http://t.co/wvkhcrr4nM #SportsRoadhouse\n",
        "[ 0.0 ]  Cooling it with my kid . . . #LongAsMyBitchesLoveMe #Honey #Chilling #UCLA http://t.co/sleK30SlGn\n",
        "[ 0.0 ]  if i get into UCLA, i would had accomplished my number 1 goal in life\n",
        "[ 0.0 ]  Top 1% of the student body, highest academic honor UCLA givessssss. can't freaken believe it even after 2 years.\n",
        "[ 1.0 ]  Report: Vanderdoes to UCLA - Five-star defensive tackle Eddie Vanderdoes plans to enroll at UCLA this month, will ... http://t.co/SigYITIiXl\n",
        "[ 1.0 ]  Stanford has this on the entrance of campus, and I want UCLA to have one a lot like it. http://t.co/yM5MDqoRHM\n",
        "[ -1.0 ]  Father of former UCLA basketball star indicted http://t.co/AFH1kgrCqx\n",
        "[ 1.0 ]  Ryan McDermott Performs at the UCLA Jazz &amp; Reggae Festival:  G.O.O.D. Music's own Ryan McDermott took the ... http://t.co/BAakSglG93\n",
        "[ 1.0 ]  Los Angeles Regional Preview: #UCLA Favorites In An All-California Regional http://t.co/FrVqW80foq\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1.0 ]  I'm at UCLA Tiverton House (Los Angeles, CA) http://t.co/EEFAQk6KuL\n",
        "[ 1.0 ]  UCLA Study: \u201cJobs are growing, but not rapidly enough to create good jobs for all.\u201d http://t.co/xiiS1TlenA #tcot\n",
        "[ 0.0 ]  Secretly feel bad for UCLA&amp;USC students bcs we get all the same professors, but pay 1/10 of their tuition.\n",
        "[ 0.0 ]  At UCLA, freshman 15 should not exist. Should be losing weight if anything, all these dam stairs and hills!\n",
        "[ 0.0 ]  A lil rugby on the tube... Yea I called it the tube #College7s #UCLA #Cal http://t.co/47bwxaIgt7\n",
        "[ 0.0 ]  \u201c@katiieepoo: @laurensteinbeck no I'm pretty sure he loves me mostest!! UCLA BABY!!\u201d I WANT TO GO TO UCLA TOO YA SKANK\n",
        "[ 0.0 ]  Follow UCLA Offensive Coordinator @NzoneFootball for camp info June 8-9! Register @ http://t.co/EMPhRpO7Jv #Q90Camp\n",
        "[ 0.0 ]  #UCLA coach John Savage on the left, SDSU coach Tony Gwynn (heard of him?) on the right. #NCAARegional http://t.co/vwmA7CjrZ3\n",
        "[ 1.0 ]  Softball. Yvonne Gutierrez Named to UCLA Athletics Hall of Fame http://t.co/faMjlroLt7\n",
        "[ 0.0 ]  @geny1020 haha he needs a tan tho! Yeah, by UCLA..\n",
        "[ 0.0 ]  @LombardTrucking yup yup in Philly. UCLA vs. Navy was just on. Navy lost 7-12\n",
        "[ 0.0 ]  Lisa Teasley '84 was the guest Sunday at the Idyllwild Authors Series, hosted by UCLA Extension's Eduardo... http://t.co/0rE91sDO3R\n",
        "[ 0.0 ]  Heriza's book hlps reaffirm role ofYOGA as a serious thrputic modality.\"PK Shah,MD.Dir,Cardio,CSMC;ProfMed,UCLA http://t.co/IlGrhiyHfz\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.0 ]  5-star forward Justice Winslow visiting #UCLA. RT: @Chief_Justise Got some late night shots up tonight... #Westwood http://t.co/FMRvrVaht4\n",
        "[ 0.0 ]  Irish DT recruit Vanderdoes going to UCLA http://t.co/iy6foO61Ni\n",
        "[ 0.0 ]  UCLA forecasters still a gloomy bunch - LA Biz Observed: \n",
        "      Blog Category:\u00a0\n",
        "\n",
        "            ... http://t.co/uLMYCAXyVm\n",
        "[ 0.3 ]  Eddie Vanderdoes Will Be Well Worth the Wait for UCLA Bruins http://t.co/q3Vbfp05rC\n",
        "[ 0.0 ]  \u201c@JessicaS0T0: I NEED to be at that UCLA, Fullerton game this week. Definitely a must!\u201d I NEED Fullerton to win!!\n",
        "[ 0.0 ]  Game over. UCLA. 8-1\n",
        "[ 0.0 ]  #doublepints #dunzo #UCLA https://t.co/npbw6Is5Vh\n",
        "[ 0.0 ]  Reilly walks the nine-place hitter, putting runners on first and second for UCLA. One out. CP 4, UCLA 4. Seventh inning. #CPBats\n",
        "[ 0.0 ]  shout out to #UCSB #UCLA #SLO for rejecting me 3 years ago. thank you.\n",
        "[ 1.0 ]  #tcot LAT Story on UCLA Study Relayed By AP: 'It's Not a Recovery. It's Not Even Normal Gr... http://t.co/Os1q4hmS44 #teaparty #sgp #gop\n",
        "[ 1.0 ]  @Helens_Brail @udubtrailblazer it's official, Eddie Vanderdoes to UCLA.\n",
        "[ 0.0 ]  Would be a HUGE get. Immediately makes UCLA a PAC-12 contender if they weren't already\n",
        "[ 0.3 ]  #UCLA Eddie Vanderdoes Officially Leaving Notre Dame for UCLA Bruins - Bleacher Report http://t.co/rnCUsA4XqF #SportsRoadhouse\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1.0 ]  Wanna Bail!!!! \u201c@JonRothstein: Wannah Bail gives Steve Alford another capable rotation player next season at UCLA.\u201d\n",
        "[ -1.0 ]  Former UCLA star's dad under house arrest: @ESPN http://t.co/Id2kdv7GFo #NBA\n",
        "[ 0.0 ]  Yo, #rugby is quite legit. Just saw J. Anderson of #Cal hurdling someone from #UCLA\n",
        "[ 1.0 ]  Photoset: thegoodlittlesoldier: Jensen and his adorable smile at the UCLA Spring Sing event. http://t.co/2XVOhL1vfn\n",
        "[ 0.8 ]  Getting ready for the presentation #IYEC #UCLA @ UCLA Powell Library http://t.co/Icy1QydGVj\n",
        "[ 0.0 ]  I can't believe my cousin is graduating 4th academically at Ike! So proud of her! UCLA here she comes!\n",
        "[ 0.0 ]  UCLA\u2019s flyer on digitized medical records: Check out THIS guy, looking at x-rays from\u2026poolside at the Four Seasons?? http://t.co/cGiO1Qqg7S\n",
        "[ 1 , 1.0 ]  Ah! YAY! This is def what's uppp :) \u201c@LAWeekly: Westwood Sucks, So These UCLA Bands Made Their Own Scene http://t.co/j80agChT8x\u201d\n",
        "[ 1.0 ]  Photoset: itsajensenthing: Jensen Ackles at UCLA Spring Sing [x] (please don\u2019t change source) http://t.co/f31J3QsshK\n",
        "[ 0.0 ]  I remember being sat on the couch watching the news..Michael had just passed, and all #MJFam and fans were outside the UCLA..urgh #flashback\n",
        "[ 0.7 ]  #thedreamisnow @ UCLA James Bridges Theater http://t.co/NNMfe7HQgZ\n",
        "[ 0.0 ]  @lupedupe Did you know UCLA played Navy in a Rugby match Saturday?  I couldn't watch the whole thing sadly.\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1.0 ]  Ryan McDermott - UCLA Jazz &amp; Reggae Festival | @RyanMcDMusic - View Post http://t.co/NK5CNZxcyX\n",
        "[ 0.0 ]  Going out w a chick who is a senior at UCLA was the clear winner in best thing about the weekend\n",
        "[ 1.0 ]  Eddie Vanderdoes: Notre Dame signee chooses UCLA - SportingNewscom http://t.co/Qcj9ZMRdqe #NotreDame #FightingIrish\n",
        "[ 0.0 ]  Toxic Twins Groupon And Zynga Slide Again Friday http://t.co/Aoh0cXZgPs\n",
        "[ -0.1 ]  @sheloveslifexox I'm not leaving for a couple years. I'm going to slcc then ucla\n",
        "[ 0.0 ]  Bijzonder: \"UCLA Broadcasts Brain Surgery Live to Twitter, Vine, &amp; Instagram\" http://t.co/MBgRUnvMAr #gen-y\n",
        "[ -1.0 ]  A federal magistrate in Las Vegas has ordered the father of former UCLA basketball star Shabazz Muhammad held... http://t.co/Bi6uUQWj40\n",
        "[ 0.0 ]  UCLA Scientists Isolate And Characterize New Population Of StressResistant Pluripotent Stem Cells In Fat Tissue Re... http://t.co/SRrNdILbRG\n",
        "[ 0.0 ]  Come to the Biomedical Library on TUESDAY between noon and 2 PM! for FREE massages to students as part of UCLA Library Stressbusters.\n",
        "[ 0.4 ]  FINAL: #Fullerton 6, #ASU 1 - Cal State Fullerton wins the Fullerton Regional - Will host UCLA next weekend in the super regionals.\n",
        "[ 0.0 ]  Listen to @TravisRodgers' interview with @FullertonTitans' Rick Vanderhook http://t.co/GQOd5nutxx #AM830 #UCLA\n",
        "[ 0.0 ]  OMG HAHA, Kim told me she wants to get faded with me before she leaves for UCLA. HAHAHA. #lovethatnigga #tu\n",
        "[ 1.0 ]  #Google #Top #News 5-star Eddie Vanderdoes transfers to UCLA after leaving Notre Dame http://t.co/zBDf02lDdi #InstantFollowBack GTNews\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.1 ]  @HoyosRevenge once schools like UCLA, Arizona, North Carolina, etc come a knocking we don't stand a chance.\n",
        "[ 0.0 ]  Smeagol's gonna have a blast being my roommate for the UCLA camp. \ud83d\ude08\n",
        "[ 0.0 ]  Asians girls everywhere, UCLA &gt; _&lt;\n",
        "[ 0.0 ]  Quick Hives Solution In Ucla http://t.co/2eFwgMJcFO\n",
        "[ 0.0 ]  Had a Legend show up to UCLA recruiting trip head to toe in (free) Michigan gear. He committed to the U though so it was whatevs\n",
        "[ -1.0 ]  Former UCLA star's dad under house arrest: A federal magistrate on Friday ordered the father of former UCLA ba... http://t.co/iG5wKXAccB\n",
        "[ 1.0 ]  @omgnarry @cloudyzouis someone asked what college ur going to and i said pasadena city college then ucla (is that right yeah)\n",
        "[ 0.4 ]  So ND doesnt release Vanderdoes from NLI because they dont play UCLA for the 4 years un-believable--NCAA strikes again. #freeEVanderdoes\n",
        "[ 0.0 ]  And then there was the UCLA professor who opened up his vest, pulled out his tie and wet his pants.\n",
        "[ 0.0 ]  UCLA researchers have found that bacteria ingested in food can affect #brain function in humans http://t.co/OUs9pDNA2C\n",
        "[ -1 , -0.6 ]  Do I stay at Pcc for another WHOLE year and see IF I make it to UCLA or just go to cal state in the spring...damn:/ #decisions\n",
        "[ 0.0 ]  Show Force One is on the way to UCLA. #ShowOnTheRoad\n",
        "[ 0.0 ]  Video: Angelica Felix fulfills dream to play for UCLA  http://t.co/LTq2HmgDRB via http://t.co/lt1jPDz9PV\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.0 ]  There's a course in UCLA called Godzilla to Hello Kitty: Japanese Popular Culture.\n",
        "\n",
        "Why can't we have nice things like this?\n",
        "[ 0.0 ]  UCLA scores!\n",
        "[ 0.0 ]  @astrofrog @davidwhogg @rahuldave I had success getting students to estimate the mass of the SMBH using the UCLA gif: http://t.co/1d1mtv17i1\n",
        "[ 0.0 ]  UCLA Reinforces SoCal 'Juggernaut' Status - http://t.co/f9kAyIU8ie: http://t.co/ZETd0ocxvY Bruins in the Super-Regionals vs. CSUF...\n",
        "[ 0.0 ]  UCLA researchers + Gallup poll: Healthy lifestyle habits may improve your memory too http://t.co/4S2dXN7ivO via @USATODAY\n",
        "[ 0.0 ]  \u201c@MrBruin2k: Zqtwzdozqwxr\u201d. Did UCLA sign someone new?? LOL\n",
        "[ 0.0 ]  @SGVtylerd Assigned seating unfortunately. I'm sure they have one behind the UCLA clubhouse for you though.\n",
        "[ 0.0 ]  I'm featured today in UCLA's Daily Bruin! @OdysseyTheatre_ #IONESCOPADE @odyssey http://t.co/K8mYh5gcSi\n",
        "[ 1.0 ]  @SacBee_JoeD I call bullshit by UCLA how much u want to bet they were still contacting him while he was at ND can u say major violation\n",
        "[ 1 , 0.4 ]  @phonkdivany cal state La / long beach , or UCLA(:\n",
        "[ 0.3 ]  #UCLA Eddie Vanderdoes Will Be Well Worth the Wait for UCLA Bruins - Bleacher Report http://t.co/ZDKY3bzp85 #SportsRoadhouse\n",
        "[ -1.0 ]  Hmmm guess I will go to UCLA tonight\n",
        "[ 0.0 ]  About to start reading for my sociology class #PhDgrind #highered  #ucla\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.0 ]  So cool! #Ucla http://t.co/eLZC4kC3mo\n",
        "[ 1.0 ]  And when I sneak into UCLA I'll be considered a college student if I'm thinner\n",
        "[ 0.3 ]  #cfb Eddie Vanderdoes Officially Leaving Notre Dame for UCLA Bruins http://t.co/3mBo7Ac23E\n",
        "[ 0.0 ]  I love my school #UCLA #Forever http://t.co/Zbq07khMZs\n",
        "[ 0.3 ]  UCLA Bruins Leather Theater 2 Seats with Curved Configuration: For the ultimate game day experience, try our C... http://t.co/dn94DGxZAX\n",
        "[ 0.0 ]  BUT UCLA IS SO FuckinG PERFECT AND BEAUTIFUL AND OMG I WanT TO GO SO BAD BUT I DONT THINK ILL GET ACCEPTED BECAUSE THE GPA IS 4.2 AND I+\n",
        "[ 0.0 ]  @davidberg_26 strikes out the side in B9 for the save and UCLA moves onto the winners bracket. #RoadtoOmaha\n",
        "[ 1.0 ]  Congrats to@uclamsoccer's Carlos Bocanegra for his induction into the UCLA Athletics Hall of Fame http://t.co/bBcKiUv04w#GoBruins#ALJHS#ALHS\n",
        "[ 0.0 ]  Notre Dame recruit reneges http://t.co/dLTgT9Jkmg\n",
        "[ 0.0 ]  Cal Poly starter Matt Imhof had no-decision in 6-4 winners-bracket loss at UCLA: 5.1-4R-3H-BB-HB-5K\n",
        "[ 0.0 ]  @BHugh215 Yes. UCLA whooped them 66-10.\n",
        "[ 0.0 ]  Kyle Bosworth was undrafted out of UCLA, but has already played longer as a pro than Uncle Brian: http://t.co/qXiLqAJ1U4 #Giants\n",
        "[ 0.7 ]  My UCLA Anderson class. Final business plan presentations. @ UCLA Anderson School of Management http://t.co/BJFUjcwfdn\n",
        "[ "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1.0 ]  Justine Bateman is a freshman at UCLA and she's studying Computer Science.  http://t.co/5dPFZWwmUl\n",
        "[ 1.0 ]  Vanderdoes jilts Notre Dame for UCLA: Eddie Vanderdoes, the nation's top-rated defensive tackle recruit in... http://t.co/RvOXsJa7Jr\n",
        "[ 0.0 ]  Getting stuff together for my two week trip to Phoenix and LA! Excited to see @Meg_Gawlik and @MSUAAF! #NSAC #AAF #ucla #vacation\n",
        "[ 0.0 ]  @HOTARUMYST i have friends in UCLA and in pennsylvania tho lol\n",
        "[ 0.0 ]  #Yahoo #Trending #Now Irish signee Vanderdoes switches to UCLA instead http://t.co/GQn2jtL8Zp #F4F Woo\n",
        "[ -0.0 ]  BSB NCAA Regionals Saturday: #UCSB vs Oregon State, 5pm PT (ESPNU/3) &amp; #CalPoly vs UCLA, 6pm (ESPN3 only).\n"
       ]
      }
     ],
     "prompt_number": 107
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def train_all_classifiers(nb, knn, svm, tweet_set, positive, negative, extract=get_tokens):\n",
      "    sentiment = positive.union(negative)\n",
      "    for tweet in tweet_set:\n",
      "        tokens = set([val.lower() for val in tweet['raw_tokens']])\n",
      "        if tokens.intersection(sentiment):\n",
      "            #print tweet['text']\n",
      "            polarity = 0\n",
      "            if tokens.intersection(positive): polarity += 1\n",
      "            if tokens.intersection(negative): polarity -= 1\n",
      "            if polarity != 0:\n",
      "                #print \"    \",tokens.difference(sentiment)\n",
      "                t = extract(tweet, filter='default', exclude=sentiment)\n",
      "                v = Document(t, stopwords=False, type=int(polarity))\n",
      "                #v = Document(list(tokens.difference(sentiment)), stopwords=False, type=int(polarity))\n",
      "                nb.train(v)\n",
      "                knn.train(v)\n",
      "                svm.train(v)\n",
      "\n",
      "def compare_tweet_classification(nb, knn, svm, tweet_set, positive, negative, extract=get_tokens):\n",
      "    sentiment = positive.union(negative)\n",
      "    for tweet in tweet_set:\n",
      "        flag = False\n",
      "        #print '[',\n",
      "        tokens = set([val.lower() for val in tweet['raw_tokens']])\n",
      "        if tokens.intersection(sentiment):\n",
      "            flag = True\n",
      "            #print '[',\n",
      "            polarity = 0\n",
      "            if tokens.intersection(positive): polarity += 1\n",
      "            if tokens.intersection(negative): polarity -= 1\n",
      "            print 'TRAIN:', polarity\n",
      "        \n",
      "        t = extract(tweet, filter='default', exclude=sentiment)\n",
      "        v = Document(t, stopwords=False)\n",
      "        #v = Document(list(tokens), stopwords=False)\n",
      "        pnb = nb.classify(v)\n",
      "        pnb = pnb[1] - pnb[-1]\n",
      "        pknn = knn.classify(v)\n",
      "        pknn = pknn[1] - pknn[-1]\n",
      "        psvm = svm.classify(v)\n",
      "        psvm = psvm[1] - psvm[-1]\n",
      "        p = pnb+pknn+psvm\n",
      "        #if flag or p == 3 or p == -3: print '[', pnb, ',', pknn, ',', psvm, '] ', tweet['text']\n",
      "        print '[', \"%.1f\" % pnb, ',', \"%.1f\" % pknn, ',', \"%.1f\" % psvm, '] ', tweet['text']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 72
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "nb = NB()\n",
      "knn = KNN()\n",
      "svm = SVM()\n",
      "\n",
      "train_all_classifiers(nb, knn, svm, tweet_set, positive, negative, extract=get_lemmatize_bigrams)\n",
      "#compare_tweet_classification(nb, knn, svm, tweet_set[:200], positive, negative, extract=get_lemmatize_bigrams)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 125
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#help(Document)\n",
      "#help(NB)\n",
      "#help(KNN)\n",
      "#help(SVM)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}