特征选择方法详解

特征选择是机器学习中提升模型性能和可解释性的关键技术。通过选择最相关的特征，可以减少过拟合、加快训练速度、提高模型可解释性。

数据准备

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

# 生成高维数据
X, y = make_classification(
    n_samples=1000,
    n_features=50,
    n_informative=10,
    n_redundant=10,
    n_classes=2,
    random_state=42
)

feature_names = [f'feature_{i}' for i in range(X.shape[1])]
X_df = pd.DataFrame(X, columns=feature_names)

# 数据集划分
X_train, X_test, y_train, y_test = train_test_split(
    X_df, y, test_size=0.2, random_state=42
)

print(f"特征数量: {X_df.shape[1]}")
print(f"训练集大小: {X_train.shape[0]}")

过滤法（Filter Methods）

过滤法基于统计指标对特征进行评分，独立于模型：

from sklearn.feature_selection import (
    SelectKBest, f_classif, mutual_info_classif,
    chi2, VarianceThreshold
)

# 1. 方差过滤（去除低方差特征）
var_selector = VarianceThreshold(threshold=0.5)
X_var = var_selector.fit_transform(X_train)
selected_features_var = X_train.columns[var_selector.get_support()]
print(f"方差过滤后特征数: {len(selected_features_var)}")

# 2. 相关系数法
from scipy.stats import pearsonr

correlations = []
for col in X_train.columns:
    corr, _ = pearsonr(X_train[col], y_train)
    correlations.append({'特征': col, '相关系数': abs(corr)})

corr_df = pd.DataFrame(correlations).sort_values('相关系数', ascending=False)
print("\n相关系数最高的10个特征:")
print(corr_df.head(10))

# 3. 卡方检验（仅适用于非负特征）
X_train_nonneg = X_train - X_train.min()
selector_chi2 = SelectKBest(chi2, k=10)
X_chi2 = selector_chi2.fit_transform(X_train_nonneg, y_train)
selected_features_chi2 = X_train.columns[selector_chi2.get_support()]
print(f"\n卡方检验选择的特征: {list(selected_features_chi2)}")

# 4. 互信息
selector_mi = SelectKBest(mutual_info_classif, k=10)
X_mi = selector_mi.fit_transform(X_train, y_train)
selected_features_mi = X_train.columns[selector_mi.get_support()]
print(f"互信息选择的特征: {list(selected_features_mi)}")

# 5. F检验
selector_f = SelectKBest(f_classif, k=10)
X_f = selector_f.fit_transform(X_train, y_train)
selected_features_f = X_train.columns[selector_f.get_support()]
print(f"F检验选择的特征: {list(selected_features_f)}")

可视化特征重要性

# 可视化不同方法的特征重要性
fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# F检验得分
f_scores = pd.Series(selector_f.scores_, index=X_train.columns)
f_scores.nlargest(15).plot(kind='barh', ax=axes[0, 0])
axes[0, 0].set_title('F检验得分')

# 互信息得分
mi_scores = pd.Series(selector_mi.scores_, index=X_train.columns)
mi_scores.nlargest(15).plot(kind='barh', ax=axes[0, 1])
axes[0, 1].set_title('互信息得分')

# 相关系数
corr_df.head(15).plot(kind='barh', x='特征', y='相关系数', ax=axes[1, 0])
axes[1, 0].set_title('相关系数')

plt.tight_layout()
plt.show()

包裹法（Wrapper Methods）

包裹法使用模型性能作为评价标准：

from sklearn.feature_selection import RFE, RFECV
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

# 1. 递归特征消除（RFE）
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rfe = RFE(estimator=rf, n_features_to_select=10, step=5)
rfe.fit(X_train, y_train)

selected_features_rfe = X_train.columns[rfe.support_]
print(f"RFE选择的特征: {list(selected_features_rfe)}")

# 2. 交叉验证RFE
rfecv = RFECV(
    estimator=rf,
    step=5,
    cv=5,
    scoring='accuracy',
    n_jobs=-1
)
rfecv.fit(X_train, y_train)

print(f"最佳特征数量: {rfecv.n_features_}")
selected_features_rfecv = X_train.columns[rfecv.support_]
print(f"RFECV选择的特征: {list(selected_features_rfecv)}")

# 绘制特征数量与交叉验证分数的关系
plt.figure(figsize=(8, 5))
plt.plot(range(1, len(rfecv.cv_results_['mean_test_score']) + 1),
         rfecv.cv_results_['mean_test_score'])
plt.xlabel('特征数量')
plt.ylabel('交叉验证准确率')
plt.title('RFECV: 特征数量 vs 准确率')
plt.grid(True, alpha=0.3)
plt.show()

嵌入法（Embedded Methods）

嵌入法在模型训练过程中自动进行特征选择：

from sklearn.linear_model import LassoCV
from sklearn.feature_selection import SelectFromModel

# 1. L1正则化（Lasso）
lasso = LassoCV(cv=5, random_state=42)
lasso.fit(X_train, y_train)

# 获取特征重要性
lasso_coef = pd.Series(np.abs(lasso.coef_), index=X_train.columns)
selected_features_lasso = lasso_coef[lasso_coef > 0].index
print(f"Lasso选择的特征数: {len(selected_features_lasso)}")

# 2. 基于树模型的特征选择
selector_tree = SelectFromModel(
    RandomForestClassifier(n_estimators=100, random_state=42),
    threshold='median'
)
selector_tree.fit(X_train, y_train)
selected_features_tree = X_train.columns[selector_tree.get_support()]
print(f"树模型选择的特征数: {len(selected_features_tree)}")

# 3. 基于XGBoost的特征选择
from xgboost import XGBClassifier

xgb = XGBClassifier(n_estimators=100, random_state=42, use_label_encoder=False, eval_metric='logloss')
xgb.fit(X_train, y_train)

xgb_importance = pd.Series(xgb.feature_importances_, index=X_train.columns)
print("\nXGBoost特征重要性Top10:")
print(xgb_importance.nlargest(10))

方法对比

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, f1_score

# 定义特征选择方法
methods = {
    '全部特征': X_train.columns.tolist(),
    'F检验': list(selected_features_f),
    '互信息': list(selected_features_mi),
    'RFE': list(selected_features_rfe),
    'Lasso': list(selected_features_lasso),
    '树模型': list(selected_features_tree)
}

results = []
for name, features in methods.items():
    if len(features) == 0:
        continue
    
    # 训练模型
    rf = RandomForestClassifier(n_estimators=100, random_state=42)
    rf.fit(X_train[features], y_train)
    
    # 预测
    y_pred = rf.predict(X_test[features])
    
    results.append({
        '方法': name,
        '特征数': len(features),
        '准确率': accuracy_score(y_test, y_pred),
        'F1分数': f1_score(y_test, y_pred)
    })

results_df = pd.DataFrame(results)
print("\n特征选择方法对比:")
print(results_df.to_string(index=False))

综合特征选择策略

from collections import Counter

# 统计每个特征被选择的次数
all_selected = (
    list(selected_features_f) +
    list(selected_features_mi) +
    list(selected_features_rfe) +
    list(selected_features_lasso) +
    list(selected_features_tree)
)

feature_counts = Counter(all_selected)
feature_counts_df = pd.DataFrame(
    list(feature_counts.items()),
    columns=['特征', '被选择次数']
).sort_values('被选择次数', ascending=False)

print("特征被选择次数统计:")
print(feature_counts_df.head(15))

# 选择被多数方法选中的特征
common_features = feature_counts_df[feature_counts_df['被选择次数'] >= 3]['特征'].tolist()
print(f"\n被3种以上方法选中的特征: {common_features}")

# 使用综合特征训练模型
rf_final = RandomForestClassifier(n_estimators=100, random_state=42)
rf_final.fit(X_train[common_features], y_train)
y_pred_final = rf_final.predict(X_test[common_features])
print(f"综合特征选择后准确率: {accuracy_score(y_test, y_pred_final):.4f}")

总结

特征选择三大方法各有优劣：过滤法速度快但忽略特征交互；包裹法效果好但计算量大；嵌入法兼顾效率和效果。实际项目建议：先用过滤法快速筛选，再用嵌入法精细选择。综合多种方法的结果往往能获得最佳特征子集。