学习：利用python进行数据分析

写代码，还是要多写。自认为看懂了，可是写还是写不出来的

# -*- coding: utf-8 -*-
"""
Created on Wed May 10 20:12:39 2017

@author: caozb
"""

path=r'D:\pyd-book\ch02\usagov_bitly_data2012-03-16-1331923249.txt'
open(path).readline()
import json
path=r'D:\pyd-book\ch02\usagov_bitly_data2012-03-16-1331923249.txt'
records=[json.loads(line) for line in open(path)]
time_zones=[rec['tz'] for rec in records if 'tz' in rec]
time_zones[:10]
def get_counts(sequence):
    counts={}
    for x in sequence:
        if x in counts:
            counts[x]+=1
        else:
            counts[x]=1
    return counts

from collections import defaultdict
def get_counts2(sequence):
    counts=defaultdict(int)
    for x in sequence:
        counts[x]+=1
    return counts

def top_counts(count_dict,n=10):
    value_key_pairs=[(count,tz) for tz,count in count_dict.items()]
    value_key_pairs.sort()
    return value_key_pairs[-n:]


from collections import Couter
counts=Counter(time_zones)
counts.most_common(10)

from pandas import DataFrame,Series
import pandas as pd;import numpy as np 
frame=DataFrame(records)

clean_tz=frame['tz'].fillna('Missing')
clean_tz[clean_tz=='']='Unkown'
tz_counts=clean_tz.value_counts()
tz_counts[:10]
tz_counts[:10].plot(kind='barh',rot=0)
results=Series([x.split()[0] for x in frame.a.dropna() ])
results[:10]
results.value_counts()[:8]
cframe=frame[frame.a.notnull()]
operating_system=np.where(cframe['a'].str.contains('Windows'),'Windows','Not Windows')

by_tz_os=cframe.groupby(['tz',operating_system])

agg_counts=by_tz_os.size().unstack().fillna(0)
agg_counts[:10]

indexer=agg_counts.sum(1).argsort()
indexer[:10]

count_subset=agg_counts.take(indexer)[-10:]
count_subset.plot(kind='barh',stacked=True)
normed_subset=count_subset.div(count_subset.sum(1),axis=0)
normed_subset.plot(kind='barh',stacked=True)

　

import pandas as pd 
unames=['user_id','gender','age','occupation','zip']
users=pd.read_table(r'D:\pyd-book\ch02\movielens\users.dat',sep='::',header=None,names=unames)
rnames=['user_id','movie_id','rating','timestamp']
ratings=pd.read_table(r'D:\pyd-book\ch02\movielens\ratings.dat',sep='::',header=None,names=rnames)
mnames=['movie_id','title','genres']
movies=pd.read_table(r'D:\pyd-book\ch02\movielens\movies.dat',sep='::',header=None,names=mnames)

##验证
users[:3]
ratings[:3]
movies[:3]


data=pd.merge(pd.merge(ratings,users),movies)
data


mean_ratings=pd.DataFrame.pivot_table(data,values='rating',index=['title'],columns=['gender'],aggfunc='mean')
mean_ratings[:5]
ratings_by_title=data.groupby('title').size()
ratings_by_title[:10]

active_titles=ratings_by_title.index[ratings_by_title>=250]
active_titles
mean_ratings=mean_ratings.ix[active_titles]

top_female_ratings=mean_ratings.sort_index(by='F',ascending=False)
top_female_ratings[:10]


mean_ratings['diff']=mean_ratings['M']-mean_ratings['F']
sorted_by_diff=mean_ratings.sort_index(by='diff')
sorted_by_diff[:15]
sorted_by_diff[::-1][:15]
rating_std_by_title=data.groupby('title')['rating'].std()
rating_std_by_title=rating_std_by_title.ix[active_titles]
rating_std_by_title.order(ascending=False)[:10]

　　

import numpy as np 
from numpy.random import randn
data={i:randn() for i in range(7)}

　　

在IPython会话环境中，所有文件都可以通过%run命令当做python程序来运行。

pandas是基于numpy构建的，让以numpy为中心的应用变得更加简单