Pandas入门教程(六)

import pandas as pd
gl=pd.read_csv('./pandas/data/game_logs.csv')

# 数据的内存使用情况
gl.info(memory_usage='deep')

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 171907 entries, 0 to 171906
Columns: 161 entries, date to acquisition_info
dtypes: float64(77), int64(6), object(78)
memory usage: 859.4 MB

for dtype in ['float64','object','int64']:
    selected_dtype=gl.select_dtypes(include=[dtype])
    memory_usage_b=selected_dtype.memory_usage(deep=True).mean()
    memory_usage_mb=memory_usage_b/1024/1024
    print('[%s] memory usage %0.2f MB' % (dtype,memory_usage_mb))

[float64] memory usage 1.29 MB
[object] memory usage 9.50 MB
[int64] memory usage 1.12 MB

# uint8 int8 int16 int32 int64的取值范围
import numpy as np
for dtype in ['uint8','int8','int16','int32','int64']:
    print(np.iinfo(dtype))

Machine parameters for uint8
---------------------------------------------------------------
min = 0
max = 255
---------------------------------------------------------------

Machine parameters for int8
---------------------------------------------------------------
min = -128
max = 127
---------------------------------------------------------------

Machine parameters for int16
---------------------------------------------------------------
min = -32768
max = 32767
---------------------------------------------------------------

Machine parameters for int32
---------------------------------------------------------------
min = -2147483648
max = 2147483647
---------------------------------------------------------------

Machine parameters for int64
---------------------------------------------------------------
min = -9223372036854775808
max = 9223372036854775807
---------------------------------------------------------------

# 类型转换后的数据占用内存
def mem_usage(data):
    if isinstance(data,pd.DataFrame):
        mem_b=data.memory_usage(deep=True).sum()
    else:
        mem_b=data.memory_usage(deep=True)
    return "{:03.2f} MB".format(mem_b/1024**2)

gl_int64=gl.select_dtypes(include=['int64'])

# 向下类型转换
gl_int32=gl_int.apply(pd.to_numeric,downcast='unsigned')
print(mem_usage(gl_int64))
print(mem_usage(gl_int32))

# float64 转 float
gl_float64=gl.select_dtypes(include=['float64'])
gl_float=gl_float64.apply(pd.to_numeric,downcast='float')

print("转换前:"+mem_usage(gl_float64))
print("转换后"+mem_usage(gl_float))

7.87 MB
1.48 MB
转换前:100.99 MB
转换后50.49 MB

opt_gl=gl.copy()
opt_gl[gl_int32.columns]=gl_int32
opt_gl[gl_float.columns]=gl_float
print("原数据的大小:"+mem_usage(gl))
print("转换后的数据大小:"+mem_usage(opt_gl))

原数据的大小:859.43 MB
转换后的数据大小:802.54 MB

gl_obj=gl.select_dtypes(include=['object']).copy()
print(gl_obj.describe())

       day_of_week  v_name v_league  h_name h_league day_night  \
count       171907  171907   171907  171907   171907    140150   
unique           7     148        7     148        7         2   
top            Sat     CHN       NL     CHN       NL         D   
freq         28891    8870    88866    9024    88867     82724   

                   completion forefeit protest park_id  ... h_player_6_id  \
count                     116      145     180  171907  ...        140838   
unique                    116        3       5     245  ...          4774   
top     19590602,PIT06,2,1,39        H       V   STL07  ...      grimc101   
freq                        1       69      90    7022  ...           427   

       h_player_6_name h_player_7_id h_player_7_name h_player_8_id  \
count           140838        140838          140838        140838   
unique            4720          5253            5197          4760   
top      Charlie Grimm      grimc101   Charlie Grimm      lopea102   
freq               427           491             491           676   

       h_player_8_name h_player_9_id h_player_9_name additional_info  \
count           140838        140838          140838            1456   
unique            4710          5193            5142             332   
top           Al Lopez      spahw101    Warren Spahn            HTBF   
freq               676           339             339            1112   

       acquisition_info  
count            140841  
unique                1  
top                   Y  
freq             140841  

[4 rows x 78 columns]

dow=gl_obj.day_of_week
print(dow.head())

dow_cat=dow.astype('category')
print(dow_cat.head())

print("转换前"+mem_usage(dow))
print("转换后"+mem_usage(dow_cat))

# 将重复比较多的数据转换成category,缩小数据内存
convert_obj=pd.DataFrame()
for col in gl_obj.columns:
    num_unique=len(gl_obj[col].unique())
    num_total=len(gl_obj[col])
    if num_unique/num_total<0.5:
        convert_obj.loc[:,col]=gl_obj[col].astype('category')
    else:
        convert_obj.loc[:,col]=gl_obj[col]

print('数据转换前:'+mem_usage(gl_obj))
print('数据转换后:'+mem_usage(convert_obj))

opt_gl[convert_obj.columns]=convert_obj
print(mem_usage(opt_gl))

# apply操作

titanic=pd.read_csv('./pandas/data/titanic_train.csv')
titanic.iloc[99]

# 获取99行的数据
def get_row(data):
    return data.iloc[99]
row=titanic.apply(get_row)
row

# 统计每一列为NaN的数量
def get_null_count(data):
    col_null=pd.isnull(data)
    null=data[col_null]
    return len(null)
null_count=titanic.apply(get_null_count)
print(null_count)

# 数据转换
def which_class(row):
    pclass=row['Pclass']
    if pd.isnull(pclass):
        return "UnKown"
    elif pclass == 1:
        return "One"
    elif pclass == 2:
        return "Tow"
    elif pclass == 3:
        return "Three"
classes=titanic.apply(which_class,axis=1)
print(classes)

# 找出未成年的数据
def is_minor(row):
    age=row['Age']
    if age<18:
        return True
    else:
        return False

minor=titanic.apply(is_minor,axis=1)
print(titanic[minor])