The magic words to get free pizza

import json, re
import pandas as pd
import numpy as np

from nltk.corpus import stopwords
from sklearn.naive_bayes import MultinomialNB
from sklearn.cross_validation import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import roc_curve, auc

dfs = {}
for name in ['train', 'test']:
    df = pd.read_json('../data/%s.json' % name)
    df['_data'] = name
    dfs[name] = df

# combine train and test data into one df
df = dfs['train'].append(dfs['test'])
df = df.reset_index(drop=True)

# limit to shared columns (plus predictor)
cols = list(dfs['test'].columns) + ['requester_received_pizza']
df = df[cols]

# rename a few columns to be pithier
df.rename(columns={
        'request_title': 'title', 
        'request_text_edit_aware': 'body',
        'requester_received_pizza': 'got_pizza',
}, inplace=True)

# convert got pizza indicator to ints
df['got_pizza'] = df['got_pizza'].apply(lambda x: -1 if pd.isnull(x) else int(x))

# toss out unused columns
cols_to_keep = ['_data', 'request_id', 'title', 'body', 'got_pizza']
df = df[cols_to_keep]

df.iloc[0]

# combine title and body columns
df['txt_raw'] = df['title'] + ' ' + df['body']

# check that it worked
for col in ['title', 'body', 'txt_raw']:
    print df.iloc[0][col]
    print '--'

# clean up text (lowercase, letters only, remove stop words)

def clean_txt(raw, remove_stop=False):
    # remove non-letters
    letters_only = re.sub("[^a-zA-Z]", " ", raw) 

    # convert to lower case, split into individual words
    words = letters_only.lower().split()                             

    # remove stop words
    stops = set(stopwords.words("english"))
    words = [w for w in words if not w in stops]
    
    # join cleaned words
    return " ".join(words)

df['txt_clean'] = df['txt_raw'].apply(clean_txt)

# check that it worked
for col in ['txt_raw', 'txt_clean']:
    print df.iloc[0][col]
    print '--'

# prep training set data for modeling
# creates numerical arrays for X (bag of words) and y (got pizza)

def get_xy(vectorizer=None, txt_col='txt_clean'):
    if vectorizer is None:
        vectorizer = CountVectorizer()
        
    dg = df[df['_data'] == 'train']

    X = vectorizer.fit_transform(dg[txt_col]).toarray()
    y = dg['got_pizza'].astype(int).as_matrix()

    return X, y

# randomly split training set and try default NB model

X, y = get_xy()
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1234)

model = MultinomialNB().fit(X_train, y_train)

print "Accuracy on training data: %f" % model.score(X_train, y_train)
print "Accuracy on test data:     %f" % model.score(X_test, y_test)

y_pred = model.predict_proba(X_test)[:, 1]
fpr, tpr, thresholds = roc_curve(y_test, y_pred)
print "AUC: %f" % auc(fpr, tpr)

#
# whoa, not very good - an roc auc just a little bit better
# than random chance and a big difference in error rates
# between training and test data implying overfitting
#

# let's calibrate some params (in vectorizer & classifier)
# and choose values that maximize roc auc

# the grid of params to search over
alphas = [1, 5, 10, 25]
min_dfs = [0.001, 0.01, 0.02, 0.05]

# loop through values to find optimal settings
best_alpha, best_min_df = None, None
max_auc = -np.inf

for alpha in alphas:
    for min_df in min_dfs:
        
        vectorizer = CountVectorizer(min_df = min_df)

        X, y = get_xy(vectorizer)
        X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=123)

        model = MultinomialNB(alpha=alpha).fit(X_train, y_train)

        y_pred = model.predict_proba(X_test)[:, 1]        
        fpr, tpr, thresholds = roc_curve(y_test, y_pred)
        auc_val = auc(fpr, tpr)

        if auc_val > max_auc:
            max_auc = auc_val
            best_alpha, best_min_df = alpha, min_df 

                
print "alpha: %f" % best_alpha
print "min_df: %f" % best_min_df
print "best auc: %f" % max_auc

# let's make sure this new model is less over-fit

vectorizer = CountVectorizer(min_df = best_min_df)

X, y = get_xy(vectorizer)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1234)

model = MultinomialNB(alpha=best_alpha).fit(X_train, y_train)

print "Accuracy on training data: %f" % model.score(X_train, y_train)
print "Accuracy on test data:     %f" % model.score(X_test, y_test)

# finally, let's train on full training set with best params
# and save our predictions for submission

vectorizer = CountVectorizer(min_df = best_min_df)

X, y = get_xy(vectorizer)

model = MultinomialNB(alpha=best_alpha).fit(X, y)

df_test = df[df['_data'] == 'test'].copy()
X_test = vectorizer.transform(df_test['txt_clean'])
y_pred = model.predict_proba(X_test)[:, 1]

df_test['requester_received_pizza'] = y_pred
final_df = df_test[['request_id', 'requester_received_pizza']]

# sanity check entries 
print final_df.head(5)

# output predictions
final_df.to_csv('../output/predicted.csv', index=False)

# and for funsies, let's see which words best predict getting a pizza 

words = np.array(vectorizer.get_feature_names())

x = np.eye(X.shape[1])
probs = model.predict_proba(x)[:, 1]

word_df = pd.DataFrame()
word_df['word'] = words
word_df['P(pizza | word)'] = probs
word_df.sort('P(pizza | word)', ascending=False, inplace=True)

print 'good words'
print word_df.head(10)
print '\n---\n'
print 'bad words'
print word_df.tail(10)