Python二维数据分析

import numpy as np #每一个[]代表一行
ridership = np.array([
[ 0, 0, 2, 5, 0],
[1478, 3877, 3674, 2328, 2539],
[1613, 4088, 3991, 6461, 2691],
[1560, 3392, 3826, 4787, 2613],
[1608, 4802, 3932, 4477, 2705],
[1576, 3933, 3909, 4979, 2685],
[ 95, 229, 255, 496, 201],
[ 2, 0, 1, 27, 0],
[1438, 3785, 3589, 4174, 2215],
[1342, 4043, 4009, 4665, 3033]
])

print ridership[1,3] #取第二行,第四个数
print ridership[1:3,3:5] #取子集
print ridership[1,:]#取第二行
print ridership[0,:]+ridership[1,:]#第一行的值加上第二行的值
print ridership[:,0]+ridership[:,1]#第一列的值加上第二列的值

a=np.array([[1,2,3],[4,5,6],[7,8,9]])
b=np.array([[1,2,3],[4,5,6],[7,8,9]])
print a+b

a = np.array([
[1, 2, 3],
[4, 5, 6],
[7, 8, 9]
]) print a.sum() #求出数组所有值的和
print a.sum(axis=0) #按照行计算总和
print a.sum(axis=1) #按照列计算总和

def mean_riders_for_max_station(ridership):
max_station = ridership[0,:].argmax() #找出最大的车站的下标
overall_mean = ridership.mean() #求出总体的平均值
mean_for_max = ridership[:,max_station].mean() #根据最大的车站,找到该下标的所有列,求出平均值
return (overall_mean,mean_for_max)

def min_and_max_riders_per_day(ridership):
max_station = ridership.mean(axis=0)
max_daily_ridership = max_station.max()
min_daily_ridership = max_station.min()
return (max_daily_ridership,min_daily_ridership)

import pandas as pd
ridership_df = pd.DataFrame(
data=[[ 0, 0, 2, 5, 0],
[1478, 3877, 3674, 2328, 2539],
[1613, 4088, 3991, 6461, 2691],
[1560, 3392, 3826, 4787, 2613],
[1608, 4802, 3932, 4477, 2705],
[1576, 3933, 3909, 4979, 2685],
[ 95, 229, 255, 496, 201],
[ 2, 0, 1, 27, 0],
[1438, 3785, 3589, 4174, 2215],
[1342, 4043, 4009, 4665, 3033]],
index=['05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11',
'05-06-11', '05-07-11', '05-08-11', '05-09-11', '05-10-11'],
columns=['R003', 'R004', 'R005', 'R006', 'R007']
)

print ridership_df #以表格的方式展示
print ridership_df.iloc[0] #获取第一行的数据
print ridership_df.loc['05-05-11']#找到该下标所对应行的数据
print ridership_df['R003'] #找出该下标所对应的列的数据
print ridership_df.iloc[1,3] #根据点找出对应的值
print ridership_df[['R003','R005']] #根据多个下标获取多列的数据

df_1=pd.DataFrame({'A':[0,1,2],'B':[3,4,5]}) #默认以列的方式展示
print df_1
df_2=pd.DataFrame([[0,1,2],[3,4,5]],columns=['A','B','C']) #以行的方式展示
print df_2

def mean_riders_for_max_station(ridership):
max_station = ridership.iloc[0].argmax() #获取第一行的最大的下边,即为最繁忙的地铁站
overall_mean = ridership.values.mean() #获取数组中值的所有平均数
mean_for_max = ridership[max_station].mean() #取出平均该列的的客流量
return (overall_mean,mean_for_max)

subway_df = pd.read_csv('nyc-subway-weather.csv') #下载区有文件
def correlation(x,y):
#该函数体现了表格中两个变量之间的相关性
std_x = (x-x.mean())/x.std(ddof=0)
std_y = (y-y.mean())/y.std(ddof=0)
return (std_x*std_y).mean()

entries = subway_df['ENTRIESn_hourly'] #获取出对应列的值
cum_entries = subway_df['ENTRIESn']
rain = subway_df['meanprecipi']
temp = subway_df['meantempi'] #找出两个变量之间的关联度
print correlation(entries,rain)
print correlation(entries,temp)
print correlation(rain,temp)
print correlation(entries,cum_entries)

df1=pd.DataFrame({'a':[1,2,3],'b':[4,5,6],'c':[7,8,9]})
df2=pd.DataFrame({'a':[10,20,30],'b':[40,50,60],'c':[70,80,90]})
print df1+df2 df1 = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6], 'c': [7, 8, 9]})
df2 = pd.DataFrame({'d': [10, 20, 30], 'c': [40, 50, 60], 'b': [70, 80, 90]})
print df1 + df2 df1=pd.DataFrame({'a':[1,2,3],'b':[4,5,6],'c':[7,8,9]},index=['row1','row2','row3'])
df2=pd.DataFrame({'a':[10,20,30],'b':[40,50,60],'c':[70,80,90]},index=['row1','row2','row3'])
print df1+df2

entries_and_exits = pd.DataFrame({
'ENTRIESn': [3144312, 3144335, 3144353, 3144424, 3144594,
3144808, 3144895, 3144905, 3144941, 3145094],
'EXITSn': [1088151, 1088159, 1088177, 1088231, 1088275,
1088317, 1088328, 1088331, 1088420, 1088753]
})
#获取每小时的进出地铁的人数
def get_hourly_entries_and_exits(entries_and_exits):
return entries_and_exits-entries_and_exits.shift(1)

df = pd.DataFrame({
'a': [1, 2, 3],
'b': [10, 20, 30],
'c': [5, 10, 15]
}) def add_one(x):
return x + 1 print df.applymap(add_one)

grades_df = pd.DataFrame(
data={'exam1': [43, 81, 78, 75, 89, 70, 91, 65, 98, 87],
'exam2': [24, 63, 56, 56, 67, 51, 79, 46, 72, 60]},
index=['Andre', 'Barry', 'Chris', 'Dan', 'Emilio',
'Fred', 'Greta', 'Humbert', 'Ivan', 'James']
)
def convert_grade(grade):
if grade>=90:
return 'A'
elif grade>=80:
return 'B'
elif grade>=70:
return 'C'
elif grade>=60:
return 'D'
else:
return 'F'
def convert_grades(grades):
return grades.applymap(convert_grade)

def standardize_column(df):
return (df-df.mean())/df.std(ddof=0) #ddof表示使用贝塞尔教正参数 def standardize(df):
return df.apply(standardize_column)
print standardize(grades_df)

df = pd.DataFrame({
'a': [4, 5, 3, 1, 2],
'b': [20, 10, 40, 50, 30],
'c': [25, 20, 5, 15, 10]
}) def second_largest_column(column):
#将数据以降序进行排列,下标为1的就是第二大的数
sorted_column = column.sort_values(ascending=False)
return sorted_column.iloc[1] def senond_large(df):
return df.apply(second_largest_column)

s=pd.Series([1,2,3,4])
df=pd.DataFrame({0:[10,20,30,40],1:[50,60,70,80],2:[90,100,110,120],3:[130,140,150,160]})
print df+s #每行对应数值相加 df = pd.DataFrame({0: [10], 1: [20], 2: [30], 3: [40]})
print df+s #行相加 df = pd.DataFrame({0: [10, 20, 30, 40]})
print df+s #列相加

s = pd.Series([1, 2, 3, 4], index=['a', 'b', 'c', 'd'])
df = pd.DataFrame({
'a': [10, 20, 30, 40],
'b': [50, 60, 70, 80],
'c': [90, 100, 110, 120],
'd': [130, 140, 150, 160]
})
print df+s #默认进行行相加
print df.add(s,axis='index') #列相加,此处不匹配,显示NaN值
print df.add(s,axis='columns') #指定进行行相加

grades_df = pd.DataFrame(
data={'exam1': [43, 81, 78, 75, 89, 70, 91, 65, 98, 87],
'exam2': [24, 63, 56, 56, 67, 51, 79, 46, 72, 60]},
index=['Andre', 'Barry', 'Chris', 'Dan', 'Emilio',
'Fred', 'Greta', 'Humbert', 'Ivan', 'James']
) #计算每个学生的标准偏差
def standardize(df):
return (df-df.mean())/df.std(ddof=0) def standardize_row(df):
#1.计算每个学生考试成绩和平均成绩的差
#2.再计算每个学生的样本偏差
mean_diffs = df.sub(df.mean(axis='columns'),axis='index')
return mean_diffs.div(df.std(axis='columns',ddof=0),axis='index')

values = np.array([1, 3, 2, 4, 1, 6, 4])
example_df = pd.DataFrame({
'value': values,
'even': values % 2 == 0,
'above_three': values > 3
}, index=['a', 'b', 'c', 'd', 'e', 'f', 'g']) print example_df #根据单列进行分组
grouped_data = example_df.groupby('even')
print grouped_data.groups
#根据多列进行分组
grouped_data = example_df.groupby(['even','above_three'])
print grouped_data.groups grouped_data = example_df.groupby('even')
print grouped_data.sum() #分组后求和
print grouped_data.sum()['value'] #先求和再根据value分组
print grouped_data['value'].sum() #继续根据value分组在求和

def standardize(xs):
return (xs-xs.mean())/xs.std(ddof=0) #根据even字段分组
grouped_data=example_df.groupby('even') #根据value在分组然后计算标准差
print grouped_data['value'].apply(standardize)

%pylab inline
import seaborn as sns
subway_df = pd.read_csv('nyc-subway-weather.csv')
#根据day_week分组,然后获取平均值,最后获取ENTRIESn_hourly列的值
ridership_by_day = subway_df.groupby('day_week').mean()['ENTRIESn_hourly']
ridership_by_day.plot()

ridership_df = pd.DataFrame({
'UNIT': ['R051', 'R079', 'R051', 'R079', 'R051', 'R079', 'R051', 'R079', 'R051'],
'TIMEn': ['00:00:00', '02:00:00', '04:00:00', '06:00:00', '08:00:00', '10:00:00', '12:00:00', '14:00:00', '16:00:00'],
'ENTRIESn': [3144312, 8936644, 3144335, 8936658, 3144353, 8936687, 3144424, 8936819, 3144594],
'EXITSn': [1088151, 13755385, 1088159, 13755393, 1088177, 13755598, 1088231, 13756191, 1088275]
})
def hour_by_group(entries_and_exits):
return entries_and_exits-entries_and_exits.shift(1)
#获取每个车站的每小时的进出口
def get_hourly_entries_and_exits(entries_and_exits):
#根据UNIT字段分组,然后,获取相应的列,最后调用自定义的方法得出结论
return entries_and_exits.groupby('UNIT')['ENTRIESn','EXITSn'].apply(hour_by_group)

subway_df = pd.DataFrame({
'UNIT': ['R003', 'R003', 'R003', 'R003', 'R003', 'R004', 'R004', 'R004',
'R004', 'R004'],
'DATEn': ['05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11',
'05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11'],
'hour': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'ENTRIESn': [ 4388333, 4388348, 4389885, 4391507, 4393043, 14656120,
14656174, 14660126, 14664247, 14668301],
'EXITSn': [ 2911002, 2911036, 2912127, 2913223, 2914284, 14451774,
14451851, 14454734, 14457780, 14460818],
'latitude': [ 40.689945, 40.689945, 40.689945, 40.689945, 40.689945,
40.69132 , 40.69132 , 40.69132 , 40.69132 , 40.69132 ],
'longitude': [-73.872564, -73.872564, -73.872564, -73.872564, -73.872564,
-73.867135, -73.867135, -73.867135, -73.867135, -73.867135]
}) weather_df = pd.DataFrame({
'DATEn': ['05-01-11', '05-01-11', '05-02-11', '05-02-11', '05-03-11',
'05-03-11', '05-04-11', '05-04-11', '05-05-11', '05-05-11'],
'hour': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'latitude': [ 40.689945, 40.69132 , 40.689945, 40.69132 , 40.689945,
40.69132 , 40.689945, 40.69132 , 40.689945, 40.69132 ],
'longitude': [-73.872564, -73.867135, -73.872564, -73.867135, -73.872564,
-73.867135, -73.872564, -73.867135, -73.872564, -73.867135],
'pressurei': [ 30.24, 30.24, 30.32, 30.32, 30.14, 30.14, 29.98, 29.98,
30.01, 30.01],
'fog': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'rain': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'tempi': [ 52. , 52. , 48.9, 48.9, 54. , 54. , 57.2, 57.2, 48.9, 48.9],
'wspdi': [ 8.1, 8.1, 6.9, 6.9, 3.5, 3.5, 15. , 15. , 15. , 15. ]
}) #将相关联的量组合
def combine_dfs(subway_df,weather_df):
return subway_df.merge(weather_df,on=['DATEn','hour','latitude','longitude'],how='inner') #如果两个列不同,则要通过on_left和on_right来匹配参数

%pylab inline
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
subway_df = pd.read_csv('nyc-subway-weather.csv')
#根据经度和纬度分组,求出平均数,注意as_index会将字段本身不作为索引,避免出错
data_for_location = subway_df.groupby(['latitude','longitude'],as_index=False).mean()
#求出每小时的标准偏差,作为图片大小
scaled_entries =
data_for_location['ENTRIESn_hourly']/data_for_location['ENTRIESn_hourly'].std(ddof=0)
#根据纬度为x轴,经度为y轴,s的教正系数,做出散点图
plt.scatter(data_for_location['latitude'],data_for_location['longitude'],s=scaled_entries)