United Statemojis of Twitter

import json
import os

from collections import defaultdict
from time import time

from emoji import UNICODE_EMOJI
from IPython.core.display import display, HTML
from tweepy import OAuthHandler, Stream, StreamListener, API as TwApi

from util.misc import CARTOGRAM, SKIN_TONES, STATE_LOOKUP, STATES
from util.tfidf import tfidf

CONSUMER_KEY = os.environ.get('TW_CONSUMER_KEY')
CONSUMER_SECRET = os.environ.get('TW_CONSUMER_SECRET')
ACCESS_TOKEN_KEY = os.environ.get('TW_ACCESS_TOKEN_KEY')
ACCESS_TOKEN_SECRET = os.environ.get('TW_ACCESS_TOKEN_SECRET')

DATA_DIR = 'data'
USA_BBOX = [-175.1, 22.4, -59.8, 72.3] # via http://boundingbox.klokantech.com/

# a few helper functions

def extract_emojis(txt):
return [c for c in txt if c in UNICODE_EMOJI]

def sort_values(data):
return sorted(data.items(), key=lambda x: x[1], reverse=True)

def get_json(fname):
with open('{}/{}'.format(DATA_DIR, fname)) as f:
return json.load(f)
  
def save_to_json(data, fname):
with open('{}/{}'.format(DATA_DIR, fname), 'w') as f:
json.dump(data, f, ensure_ascii=False)

# this uses Twitter's streaming API

# for demo purposes, this only fetches 1k tweets

# (in reality, this ran over several days and collected millions of tweets)

class MyListener(StreamListener):
def **init**(self):
super().**init**()
self.ct = 0
self.started = time()

    def on_status(self, data):
        if hasattr(data, 'retweeted_status'):
            return

        try:
            with open('{}/tweets-all.json'.format(DATA_DIR), 'a') as f:
                f.write('{}\n'.format(json.dumps(data._json)))
        except Exception as e:
            print('error: {}'.format(str(e)))

        self.ct += 1
        if (self.ct % 100 == 0):
            print('🚨 {} tweets... ({} secs elapsed)'.format(
                self.ct,
                int((time() - self.started))
            ))

        # stop stream after 1k results (for demo)
        if self.ct > 1000:
            return False

    def on_error(self, status):
        print('uh-oh! ({})'.format(status))

auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN_KEY, ACCESS_TOKEN_SECRET)
api = TwApi(auth)

stream = Stream(auth, MyListener())
stream.filter(locations=USA_BBOX)

filtered = []

with open('{}/tweets-all.json'.format(DATA_DIR)) as f:
for i, line in enumerate(f):
d = json.loads(line)
emojis = extract_emojis(d['text'])

        if i % 200000 == 0:
            print('done with {}...'.format(i))

        if not len(emojis):
            continue

        filtered.append({
            'id': d['id_str'],
            'time': d['created_at'],
            'user': d['user']['screen_name'],
            'text': d['text'],
            'coordinates': d['coordinates'],
            'place': d['place'],
            'emojis': emojis,
            'emojis_names': [UNICODE_EMOJI[e] for e in emojis],
        })

print('{} tweets with emojis'.format(len(filtered)))

save_to_json(filtered, 'tweets-w-emojis.json')

data = get_json('tweets-w-emojis.json')

data[0]

state_cts, emoji_cts = defaultdict(int), defaultdict(int)
state_emoji_cts = defaultdict(lambda: defaultdict(int))

for d in data:
place = d['place']

    if not place:
        continue

    country, ptype = place['country_code'], place['place_type']
    if country != 'US' or ptype not in ['city', 'admin']:
        continue

    state = place['name']
    if ptype == 'city':
        state = STATE_LOOKUP[place['full_name'][-2:].upper()]

    if state not in STATES:
        continue

    state_cts[state] += 1
    for e in d['emojis_names']:
        if e not in SKIN_TONES:
            emoji_cts[e] += 1
            state_emoji_cts[state][e] += 1

sort_values(state_cts)[:10]

sort_values(emoji_cts)[:10]

# show most popular emojis by state

results = []

for state, emojis in sorted(state_emoji_cts.items()):
results.append({
'state': state,
'emojis': dict(emojis),
'top_emojis': sort_values(emojis)[:10],
})

for r in results[:5]:
print('{}:\n{}'.format(r['state'], r['top_emojis'][:3]))

save_to_json(results, 'emojis-by-state.json')

data = get_json('emojis-by-state.json')
counts_list = [d['emojis'] for d in data]

data_tfidf = []

for d in data:  
 counts = d['emojis']
scores = {word: tfidf(word, counts, counts_list) for word in counts}
sorted_words = sort_values(scores)

    data_tfidf.append({
        'state': d['state'],
        'top_counts': d['top_emojis'],
        'top_tfidf': sorted_words[:10],
    })

# preview results

for d in data_tfidf[:5]:
print('{}:'.format(d['state']))

    for i, result in enumerate(d['top_tfidf'][:3]):
        word, score = result
        print("\t{}. {} tf-idf: {}".format(i + 1, word, round(score, 5)))