In [1]:
import pandas as pd
train = pd.read_csv('~/Desktop/DevMaster/Hack_day/train.csv', parse_dates=True)
test = pd.read_csv('~/Desktop/DevMaster/Hack_day/test.csv', parse_dates=True)
# it is recommended that you do this as to let the date column appropriate itself nicer
#train.info()
#train.head()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set_style('whitegrid')
import warnings
warnings.filterwarnings('ignore')
from scipy import stats
from sklearn import linear_model
from sklearn import neighbors
from sklearn.metrics import mean_squared_error,accuracy_score
from sklearn import preprocessing
from math import log
In [2]:
train.head(2)
Out[2]:
In [3]:
train.info()
In [4]:
train.notnull().sum()
Out[4]:
In [5]:
test.notnull().sum()
Out[5]:
In [6]:
# Set an empty list so we can determine which variables are numerical & categorical.
numericals = []
categoricals = []
# Set an empty list so we can determine which columns to delete.
deleteColumns = []
deleteRows = []
# Set an empty list so we can jot down our notes later for the other data preparation section.
reassess = []
transforms = []
mined = []
engineered = []
In [7]:
# Here is a list of the columns.
columns = train.columns
columns
Out[7]:
In [8]:
############################
############################
############################
In [9]:
### APPEND test to train
train.tail(1)
#train.nunique()
Out[9]:
In [10]:
test.tail(1)
Out[10]:
In [11]:
full_train=train.append(test,ignore_index=True)
In [12]:
############################
############################
############################
############################
In [13]:
def age(a):
if a <= 10.0:
return 'A'
elif a <= 20.0: ## should be 16
return 'B'
elif a <= 40.0:
return 'C'
elif a <= 55.0:
return 'D'
elif a <= 100:
return 'E'
In [14]:
def fare(a):
if a <= 200.0:
return 'A'
elif a <= 200.0:
return 'B'
elif a <= 300.0:
return 'C'
elif a <= 600.0:
return 'D'
In [15]:
#full_train.Age.value_counts()
In [16]:
#train[train['Age'].isnull()]
#train[train['Embarked'].isnull()]
#train[train['Cabin'].isnull()]
#import seaborn as sns; sns.set(style="ticks", color_codes=True)
In [17]:
#g=sns.pairplot(train)
#d={'NaN':0}
fare_median = full_train.groupby(['Sex', 'Pclass']).Fare.median()
fare_median.name = 'FareMedian'
age_mean = full_train.groupby(['Sex', 'Pclass']).Age.mean()
age_mean.name = 'AgeMean'
def join(df, stat):
return pd.merge(df, stat.to_frame(), left_on=['Sex', 'Pclass'], right_index=True, how='left')
full_train['Fare'] = full_train.Fare.fillna(join(full_train, fare_median).FareMedian)
full_train['Age'] = full_train.Age.fillna(join(full_train, age_mean).AgeMean)
#a=full_train['Age'].mean()
#b=train['Cabin'].mean()
#full_train.Age.fillna(a,inplace=True)
#train['Embarked'].value_counts()
full_train.Embarked.fillna('S',inplace=True)
#train['Embarked'].mean()
#train['Cabin'].mean()
#train['Age'].fillna(4)
In [18]:
full_train.Age=full_train.Age.apply(lambda x: age(x))
In [19]:
full_train.Fare=full_train.Fare.apply(lambda x: fare(x))
In [20]:
full_train.groupby(['Sex']).mean()
Out[20]:
In [21]:
#sns.pairplot(train)
#,x_vars=['Age','Embarked'],y_vars=['Age','Embarked']
In [22]:
#full_train['Fare'].fillna(train['Fare'].median(), inplace = True)
FEATURE ENGINEERING¶
In [23]:
######################################
import string
def subst_strings(single,whole_list):
for single_str in whole_list:
if string.find(single,single_str) !=-1:
return single_str
return np.nan
title_list=['Mrs', 'Mr', 'Master', 'Miss', 'Major', 'Rev',
'Dr', 'Ms', 'Mlle','Col', 'Capt', 'Mme', 'Countess',
'Don', 'Jonkheer']
full_train['Title']=full_train['Name'].map(lambda x: subst_strings(x,title_list))
def replace_titles(x):
title=x['Title']
if title in ['Don', 'Major', 'Capt', 'Jonkheer', 'Rev', 'Col']:
return 'Mr'
elif title in ['Countess', 'Mme']:
return 'Mrs'
elif title in ['Mlle', 'Ms']:
return 'Miss'
elif title =='Dr':
if x['Sex']=='Male':
return 'Mr'
else:
return 'Mrs'
else:
return title
full_train['Age*Class']=full_train['Age']*full_train['Pclass']
full_train['Title']=full_train.apply(replace_titles,axis=1)
###########################################
full_train['FamilySize']=full_train['SibSp']+full_train['Parch']
cabin_list = ['A', 'B', 'C', 'D', 'E', 'F', 'T', 'G', 'NaN']
full_train['Deck']=full_train['Cabin'].map(lambda x: subst_strings(str(x), cabin_list))
In [24]:
full_train.Deck.fillna('C',inplace=True)
In [25]:
#full_train.Deck.value_counts()
In [26]:
full_train['Deck']
Out[26]:
In [27]:
full_train.info()
In [28]:
deleteColumns.append('PassengerId')
deleteColumns.append('Cabin')
#deleteColumns.append('Age')
afterDelete=full_train.copy()
for columns in range(len(deleteColumns)):
del afterDelete[deleteColumns[columns]]
#del Xy['PassengerId']
del afterDelete['Ticket']
del afterDelete['Name']
In [29]:
afterDelete['Pclass']=afterDelete['Pclass'].astype('str')
In [30]:
afterDelete.info()
In [31]:
#afterDelete['Deck']
In [32]:
afterDelete.head(2)
afterDelete.nunique()
afterDelete.info()
In [66]:
Xy.columns
Out[66]:
dummies¶
Sex PClass Ticket
In [33]:
Xy = pd.get_dummies(afterDelete, drop_first = True)
#Xy.head(10)
Xy.columns
Xy.head(2)
#Xy.info()
Out[33]:
In [34]:
Xy.head(2)
Out[34]:
Final Step.............¶
In [35]:
len(Xy),len(train),len(test)
Out[35]:
In [36]:
print('split at',len(train))
In [37]:
editedTrain=Xy[:891]
editedTest=Xy[891:]
In [38]:
len(editedTrain), len(editedTest)
Out[38]:
In [39]:
y=editedTrain['Survived']
X=editedTrain.copy()
del X['Survived']
In [40]:
from sklearn.cross_validation import train_test_split as tts
X_train, X_test, y_train, y_test = tts(X, y, test_size = .2, random_state = 44)
In [41]:
################################
from sklearn.linear_model import LogisticRegression,LinearRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier,AdaBoostClassifier
log = LogisticRegression()
tree = DecisionTreeClassifier()
forest = RandomForestClassifier()
gradient = GradientBoostingClassifier()
ada = AdaBoostClassifier()
#################################
In [42]:
#models = []
#models.append(('l', LogisticRegression()))
#models.append(('t', DecisionTreeClassifier()))
#models.append(('f', RandomForestClassifier()))
#models.append(('g', GradientBoostingClassifier()))
#models.append(('a', AdaBoostClassifier()))
l = log.fit(X_train, y_train)
t = tree.fit(X_train, y_train)
ff = forest.fit(X_train, y_train)
importances=forest.feature_importances_
g = gradient.fit(X_train, y_train)
a = ada.fit(X_train, y_train)
NEW NEW NEW NEW¶
In [43]:
X_train.head(3)
Out[43]:
In [44]:
type(importances)
Out[44]:
In [45]:
len(forest.feature_importances_)
Out[45]:
In [46]:
len(X_train.columns)
Out[46]:
In [65]:
afterDelete.columns
Out[65]:
In [47]:
feat = pd.DataFrame()
#plt.figure(figsize=(10,20))
#plt.title('Feature Importance')
indices=np.argsort(importances)[::-1]
feat['x']=forest.feature_importances_
feat['y']=X_train.columns
feat.sort_values(by='x',inplace=True)
feat.set_index(feat['y'],inplace=True)
feat.plot(kind='barh',figsize=(10,10))
Out[47]:
In [48]:
'''
plt.bar(range(X_train.shape[1]),importances[indices],align="center")#, color="r", yerr=std[indices], align="center")
plt.xticks(range(X_train.shape[1]),indices)
plt.xlim([-1,X_train.shape[1]])
plt.show()
'''
Out[48]:
In [49]:
#range(X_train.shape[1])
for f in range(X.shape[1]):
print("%d. feature %d (%f)" % (f + 1, indices[f], importances[indices[f]]))
In [50]:
X_train[X_train.columns[10:16]]
Out[50]:
In [51]:
###y_test
print("The score for Logistic Regression is, ", l.score(X_test, y_test))
print("The score for Decision Trees is ", t.score(X_test,y_test))
print("The score for Random Forest is ", ff.score(X_test,y_test))
print("The score for Gradient Descent is ", g.score(X_test, y_test))
print("The score for AdaBoost is ", a.score(X_test, y_test))
In [52]:
X_train['FamilySize'].value_counts()
Out[52]:
Submission¶
In [53]:
editedTest['Fare_C'].value_counts()
Out[53]:
In [54]:
del editedTest['Survived']
editedTest.info()
In [55]:
def run_this_model(mod,test):
ret=mod.predict(test)
return ret
In [56]:
res=pd.DataFrame(columns=['PassengerId','Survived'])
res.head(2)
res['PassengerId']=test['PassengerId']
Learning curves (adapted from kaggle tutorial,not needed at the moment):¶
In [57]:
from sklearn.model_selection import cross_val_score, learning_curve, validation_curve
num_folds = 7
def plot_curve(ticks, train_scores, test_scores):
import matplotlib.pyplot as plt
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
plt.figure()
plt.fill_between(ticks,
train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.1, color="b")
plt.fill_between(ticks,
test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.1, color="r")
plt.plot(ticks, train_scores_mean, 'b-', label='Training score')
plt.plot(ticks, test_scores_mean, 'r-', label='CV score')
plt.legend()
return plt.gca()
def plot_learning_curve(clf, X, y, scoring='accuracy'):
ax = plot_curve(*learning_curve(clf, X, y, cv=num_folds, scoring=scoring,
train_sizes=np.linspace(0.1,1,10), n_jobs=-1))
ax.set_title('Learning curve: {}'.format(clf.__class__.__name__))
ax.set_xlabel('Training size')
ax.set_ylabel(scoring)
#names=['l','g']
#for i in range(len(names)):
# plot_learning_curve(i,X_train,y_train)
In [58]:
import numpy as np
In [59]:
#res['Survived']=np.random.randint(2, size=418)
res['Survived']=np.random.uniform(0,1)
#res['Survived']=np.zeros(418)
#res['Survived']=res.Survived.astype(int)
In [60]:
res['Survived']
Out[60]:
In [61]:
res
Out[61]:
In [62]:
res.to_csv('Random.csv', index=False)
In [63]:
res
Out[63]:
This is repeat of the prediction above (ala .score()):¶
In [64]:
#for name,model in models:
# model.predict(editedTest)
mod=[l,t,ff,g,a]
#survived=[]
for i in mod:
survived=run_this_model(i,X_test) #X_test
type(survived)
# print i.__class__.__name__
# print len(y_test)
sc=accuracy_score(survived,y_test)
survived=run_this_model(i,editedTest)
res['Survived']=survived.astype(int)
file_name='results_'+i.__class__.__name__+'_acc'+sc.astype(str)+'.csv'
res.to_csv(file_name, index=False)
plot_learning_curve(i,X_train,y_train)
#survived=l.predict(editedTest)
#survived=t.predict(editedTest)
#survived=f.predict(editedTest)
#survived=g.predict(editedTest)
#survived=a.predict(editedTest)
