First release

README.md

@@ -1,2 +1,52 @@
-# twitter-sentiment-streaming
-Streaming sentiment from twitter
+# Twitter Streaming Sentiment
+
+[![GitHub latest](https://img.shields.io/github/tag/tom-clements/twitter-sentiment-streaming.svg)](https://github.com/tom-clements/twitter-sentiment-streaming/)
+[![MIT license](https://img.shields.io/github/license/mashape/apistatus.svg)](http://opensource.org/licenses/MIT)
+
+### Requirements
+
+packages:
+```
+pip install -r requirements.txt
+```
+Also needed:
+- Twitter Account
+- Plotly Account
+
+### Twitter Server
+
+Choose a host to scrape live tweets. For local host use 127.0.0.1:5555.
+Create a new twitter app at https://apps.twitter.com/.
+From there, click on (manage keys and access tokens) to get all four keys required below.
+
+On lines 11-15 on the server python script, add these keys to the code:
+
+```python
+socket.bind('<IP>:<PORT>')
+CONSUMER_KEY = '<CONSUMERKEY>'
+CONSUMER_SECRET = '<CONSUMERSECRET>'
+ACCESS_TOKEN = '<ACCESSTOKEN>'
+ACCESS_TOKEN_SECRET = '<ACCESSTOKENSECRET>'
+```
+### Twitter Client
+Add the connection IP and port
+```python
+socket.connect('<IP>:<PORT>')
+```
+
+### Plotly Stream
+On your plotly account, go into settings and API keys. You'll need as many API streaming tokens as data traces you want to stream.
+Add these to
+```
+User\.plotly\.credentials
+```
+as a list format.
+### Executing
+```
+python twitter_stream_server.py
+```
+in a new command instance execute the Client
+```
+python twitter_stream_client.py
+```
+plotly should then open with the streaming data.

filter_names.csv

@@ -0,0 +1,9 @@
+names,
+blue,
+orange,
+green,
+red,
+purple,
+brown,
+pink,
+grey,

requirements.txt

@@ -0,0 +1,6 @@
+zmq
+datetime
+plotly
+pandas
+twython
+vaderSentiment

twitter_stream_client.py

@@ -0,0 +1,72 @@
+import zmq
+import datetime
+import plotly.plotly as ply
+import plotly.graph_objs as go
+import plotly.tools as pls
+import pandas as pd
+import os
+
+# get plotly stream ids
+stream_ids = pls.get_credentials_file()['stream_ids']
+
+
+# set up connection to servers
+context = zmq.Context()
+socket = context.socket(zmq.SUB)
+socket.connect('<IP>:<PORT>')
+socket.setsockopt_string(zmq.SUBSCRIBE, '')
+
+# import tweet filter words
+t = pd.read_csv(r'filter_names.csv')
+filter_name = t['names'].tolist()
+
+tweet_count = []
+stream_id = []
+token = []
+trace = []
+sent = []
+st = []
+# open data from plotly stream
+for i in range(len(filter_name)):
+    token.append(stream_ids[i])
+    stream_id.append(dict(token=token[i]))
+    sent.append(0)
+    tweet_count.append(0)
+    st.append(ply.Stream(stream_ids[i]))
+    trace.append(go.Scatter(x=[], y=[], stream=stream_id[i], name=filter_name[i]))
+
+
+# plot graphs
+
+data = trace
+layout = go.Layout(
+    title='test',
+    yaxis=dict(
+        title='sentiment'
+    )
+)
+fig = go.Figure(data=data, layout=layout)
+plot_url = ply.plot(fig, filename='stream_multiple_data')
+
+for streams in st:
+    streams.open()
+
+while True:
+    # receive data from server
+    msg = socket.recv_string()
+    # get the current time
+    t = datetime.datetime.now()
+    # get the filter name from data
+    name = msg.split()[6]
+    # split data in two
+    for i in range(len(filter_name)):
+        if name == filter_name[i]:
+            # st[i].open()
+            sent[i] += float(msg.split()[4])
+            tweet_count[i] += 1
+            # print(str(t) + ' | ' + str(sent[i]) + ' | ' + name)
+            st[i].write({'x': t, 'y': float(sent[i])})
+            os.system('cls')
+            for i in range(len(filter_name)):
+                print('{0}: Tweet Count: {1}. Sentiment: {2}.'
+                      .format(filter_name[i], tweet_count[i], sent[i]))

twitter_stream_server.py

@@ -0,0 +1,59 @@
+from twython import TwythonStreamer
+import time
+import zmq
+from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
+import pandas as pd
+
+# set up twitter and server parameters
+analyser = SentimentIntensityAnalyzer()
+context = zmq.Context()
+socket = context.socket(zmq.PUB)
+socket.bind('<IP>:<PORT>')
+CONSUMER_KEY = '<CONSUMERKEY>'
+CONSUMER_SECRET = '<CONSUMERSECRET>'
+ACCESS_TOKEN = '<ACCESSTOKEN>'
+ACCESS_TOKEN_SECRET = '<ACCESSTOKENSECRET>'
+
+
+# Twitter Streaming class
+class MyStreamer(TwythonStreamer):
+    counter = 0
+
+    def on_success(self, data):
+        try:
+            if data['lang'] in ['en', 'fr', 'de', 'es', 'it', 'ru']:
+                MyStreamer.counter += 1
+                # get sentiment of tweet
+                sent = analyser.polarity_scores(data['text'])['compound']
+                for name in filter_name:
+                    # find which filter name is contained within tweet
+                    if name in data['text'].lower():
+                        # send filter name and sentiment to server
+                        print('Tweet: {c} | Sent: {s} | {n} | {lang}'
+                              .format(c=MyStreamer.counter, s=sent, n=name,
+                                      lang=data['lang']))
+                        msg = 'Tweet: {c} | Sent: {s} | {n}' \
+                            .format(c=MyStreamer.counter, s=sent, n=name)
+                        socket.send_string(msg)
+        except:
+            pass
+
+    def on_error(self, status_code, data):
+        print(status_code, data)
+        self.disconnect()
+
+
+stream = MyStreamer(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN,
+                    ACCESS_TOKEN_SECRET)
+
+# import words to filter on
+t = pd.read_csv(r'filter_names.csv')
+filter_name = t['names'].tolist()
+t = ','.join(filter_name)
+print(t)
+timeout = time.time() + 60
+# continuously filter tweets based on names defined
+while True:
+    if time.time() > timeout:
+        break
+    stream.statuses.filter(track=t)