machine learning

Data Frame Manipulation

import pandas as pd
data={'col1':[1,2,3],'col2':[3,2,1],'col3':[5,5,5]}

df=pd.DataFrame(data)

df

Create New Column using a Function

# create a function called times100
def TestValue(x):
    if x>2:
        return True
    else:
        return False

df['bigger_than_2_by_function']=df['col1'].map(TestValue)

df

Create New Column directly

df['bigger_than_2']=df['col1']>2

df

Column basic maths

df['added']=df['col1']+df['col2']+df['col3']

df

df['other_maths']=df['col1']*df['col2']/df['col3']

df

Merge/Join

I deliberatly created 2 datatables with different key names, but matching key_values. This makes the pandas a little more difficult.

import pandas as pd

people={'id':[1,2,3,4],'name':['Tom','Bob','Harry','Frank']}
jobs={'people_id':[1,2,4],'job_desc':['Web','Db','C++']}

p_df = pd.DataFrame(people)
j_df = pd.DataFrame(jobs)

p_df

j_df

#Inner Join
pd.merge(p_df,j_df,left_on='id',right_on='people_id',how=inner)

pd.merge(p_df,j_df,left_on='id',right_on='people_id',how='outer')

pd.merge(p_df,j_df,left_on='id',right_on='people_id',how='left')

pd.merge(p_df,j_df,left_on='id',right_on='people_id',how='right')

Cleaning up Certain Values

import pandas as pd
people={'id':[1,2,3,4],'name':['Tom','Bob','Harry','Frank']}
jobs={'people_id':[1,2,4],'job_desc':['Web','Db','C++']}
p_df = pd.DataFrame(people)
j_df = pd.DataFrame(jobs)
pj_df=pd.merge(p_df,j_df,left_on='id',right_on='people_id',how='outer')

pj_df

#Need to fix the job_desc and the people_id which is NaN
#But they are different types - So I will do each one seperatly

pj_df['job_desc'].notnull()

#This index then is used with the ['job_desc'] Column so where only acts in False values, and we set the value to ""

0     True
1     True
2    False
3     True
Name: job_desc, dtype: bool

pj_df['job_desc']=pj_df['job_desc'].where(pj_df['job_desc'].notnull(),"")

pj_df

pj_df['people_id']=pj_df['job_desc'].where(pj_df['people_id'].notnull(),0)

pj_df

Visualise your data

%matplotlib inline
import pandas as pd
ship=pd.DataFrame.from_csv('train.csv')

#Remove all NaN
ship['Age']=    ship['Age'].where(ship['Age'].notnull(),0)
ship['Cabin']=  ship['Cabin'].where(ship['Cabin'].notnull(),"")
ship['Embarked']=  ship['Embarked'].where(ship['Embarked'].notnull(),"")

#Round fares to £2 "chunks"
def chunk_fare(x):
    x=int(x/2)*2
    return x

from matplotlib import pyplot as plt
rounded_fare=ship['Fare'].map(chunk_fare)
unique_fares=rounded_fare.unique()
unique_fares.sort()

rf=pd.DataFrame(rounded_fare)
fare_counts=rf.groupby(by='Fare').size()

plt.bar(unique_fares,fare_counts)
#This shows the range is too Great as there were supposed to be 1/2/3 tyoes of Tickets

<Container object of 61 artists>

def chunk_fare_v2(x):
    if x<14:
        return 3
    elif x<50:
        return 2
    else:
        return 1

rounded_fare=ship['Fare'].map(chunk_fare_v2)
unique_fares=rounded_fare.unique()
unique_fares.sort()

rf=pd.DataFrame(rounded_fare)
fare_counts=rf.groupby(by='Fare').size()
plt.title("Titanic ticket class based on price")
plt.bar(unique_fares,fare_counts)

<Container object of 3 artists>

ship['Ticket_Cost']=ship['Fare'].map(chunk_fare_v2)

ship

BarGraphs to look at Data

%matplotlib inline
import pprint
import pandas as pd
ship=pd.DataFrame.from_csv('train.csv')

#Remove all NaN
ship['Age']=    ship['Age'].where(ship['Age'].notnull(),0)
ship['Cabin']=  ship['Cabin'].where(ship['Cabin'].notnull(),"")
ship['Embarked']=  ship['Embarked'].where(ship['Embarked'].notnull(),"")

#Define some functions to help me look at the data
#Round fares to £2 "chunks"
def chunk_fare(x):
    x=int(x/2)*2
    return x

def chunk_fare_v2(x):
    if x<14:
        return 3
    elif x<50:
        return 2
    else:
        return 1

def age_function(x):
    if x<14:
        return 1
    elif x<25:
        return 2
    elif x<45:
        return 3
    else:
        return 4

#Function to look at 1 Column 
def View_Col(col_data,function_to_call,title):
    """
    Expects a Column of data
    """
    new_col_data=col_data.map(function_to_call)
    new_col_data.sort_values(inplace=True)
    rf=pd.DataFrame(new_col_data)
    rf.columns=['Data']
    col_counts=rf.groupby(by='Data').size()
    unique_cols=rf['Data'].unique()
    plt.title=title
    plt.bar(unique_cols,col_counts)


c=View_Col(ship['Age'],age_function,"Data Grouped by Age Function")

c=View_Col(ship['Fare'],chunk_fare,"Data Grouped by chuck_fare")

c=View_Col(ship['Fare'],chunk_fare_v2,"Data Grouped by chuck_fare")

Selecting data by columns

#Show only people with title's 
titanic_df[titanic_df.title_code==1][titanic_df.survived==0]