嵌入法与包装法

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# 准备数据
data = pd.read_csv("./digit recognizor.csv")
x = data.iloc[:,1:] # 特征矩阵
y = data.iloc[:,0] # 目标矩阵

Embedded 嵌入法

feature_selection.SelectFromModel

from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import RandomForestClassifier as RFC
from sklearn.model_selection import cross_val_score

rfc = RFC(n_estimators=10, random_state=0)
x_embedded = SelectFromModel(rfc, threshold=0.005).fit_transform(x, y)
x_embedded.shape

(42000, 47)

rfc.fit(x,y).feature_importances_.max()

0.01276360214820271

# 画threshold学习曲线
threshlds = np.linspace(0, rfc.fit(x, y).feature_importances_.max(), 20)
score = []
for i in threshlds:
    x_embedded = SelectFromModel(rfc, threshold=i).fit_transform(x, y)
    cross_score = cross_val_score(RFC(n_estimators=10, random_state=0), x_embedded, y, cv=10).mean()
    score.append(cross_score)
plt.plot(threshlds,score)
plt.xticks(threshlds,rotation=70)
plt.show()

​

x_embedded = SelectFromModel(rfc, threshold=0.00067).fit_transform(x, y)
cross_score = cross_val_score(RFC(n_estimators=10, random_state=0), x_embedded, y, cv=10).mean()
cross_score

0.9421904761904761

# 细化学习曲线
threshlds2 = np.linspace(0, 0.0000353, 20)
score2 = []
for i in threshlds2:
    x_embedded = SelectFromModel(rfc, threshold=i).fit_transform(x, y)
    cross_score = cross_val_score(RFC(n_estimators=10, random_state=0), x_embedded, y, cv=10).mean()
    score2.append(cross_score)
plt.figure(figsize=(20,8))
plt.plot(threshlds2,score2)
plt.xticks(threshlds2,rotation=70)
plt.show()

​

​

x_embedded2 = SelectFromModel(rfc, threshold=0.00001858).fit_transform(x, y)
cross_score2 = cross_val_score(RFC(n_estimators=10, random_state=0), x_embedded2, y, cv=10).mean()
cross_score2

0.9424523809523808

# 提高n_estimators
cross_score3 = cross_val_score(RFC(n_estimators=100, random_state=0), x_embedded2, y, cv=10).mean()
cross_score3

0.9656428571428572

Wrapper 包装法

feature_selection.RFE

from sklearn.feature_selection import RFE
rfc = RFC(n_estimators=10, random_state=0)
seletor = RFE(rfc, n_features_to_select=340, step=50).fit(x, y)

seletor.support_.sum()

340

seletor.ranking_

array([10, 9, 8, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 6, 6,
5, 6, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 6, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 6, 6, 5, 4,
4, 5, 3, 4, 4, 4, 5, 4, 5, 7, 6, 7, 7, 7, 8, 8, 8,
8, 8, 8, 8, 8, 6, 7, 4, 3, 1, 2, 3, 3, 1, 1, 1, 1,
1, 3, 3, 4, 5, 5, 5, 8, 8, 9, 9, 9, 9, 8, 9, 9, 4,
4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4,
5, 5, 9, 9, 10, 10, 10, 10, 7, 4, 4, 3, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 5, 8, 10, 10, 10,
10, 9, 4, 4, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 3, 4, 10, 10, 10, 10, 9, 7, 4, 3, 2, 2,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
4, 4, 10, 9, 10, 6, 6, 4, 2, 3, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 5, 9, 10, 8, 7,
4, 5, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 2, 1, 2, 4, 10, 10, 10, 9, 7, 5, 3, 3, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 5,
5, 9, 9, 9, 7, 5, 5, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 2, 4, 5, 9, 9, 9, 9, 9, 5,
4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 4, 5, 7, 10, 10, 9, 10, 9, 4, 1, 2, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 5, 10,
9, 10, 10, 9, 7, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 2, 2, 4, 8, 9, 10, 10, 10, 5, 4,
2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 2, 3, 5, 10, 10, 10, 10, 9, 5, 4, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 4, 5, 9,
10, 10, 10, 5, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 3, 3, 4, 8, 8, 10, 10, 9, 5, 3, 3,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
3, 3, 4, 10, 10, 10, 10, 8, 4, 3, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 4, 5, 8, 10, 10,
10, 10, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 2, 4, 7, 10, 10, 10, 10, 8, 5, 3, 2, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3,
5, 5, 7, 9, 9, 9, 9, 5, 5, 2, 2, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 4, 5, 5, 8, 9, 9, 9,
9, 7, 4, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
3, 3, 3, 5, 5, 9, 8, 9, 9, 9, 9, 9, 5, 4, 4, 2, 2,
1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 2, 3, 4, 5, 5, 9,
8, 8, 8, 8, 8, 8, 7, 8, 6, 4, 2, 2, 1, 1, 2, 2, 1,
2, 2, 3, 2, 2, 4, 4, 5, 5, 8, 8, 8, 7, 7, 7, 7, 7,
7, 7, 5, 5, 4, 5, 4, 3, 3, 3, 4, 3, 3, 4, 3, 4, 5,
5, 6, 7, 7, 7, 6, 7, 8, 8, 8, 9, 9, 9, 9, 6, 8, 8,
8, 7, 8, 8, 8, 7, 8, 8, 8, 8, 8, 7, 8, 8, 8, 8, 9,
10, 7])

x_wrapper = seletor.transform(x)

cross_score4 = cross_val_score(rfc, x_wrapper, y, cv=10).mean()
cross_score4

0.9418095238095239

cross_score5 = cross_val_score(RFC(n_estimators=100, random_state=0), x_wrapper, y, cv=10).mean()
cross_score5

0.9651190476190477