演習：離反予測モデルのスコアを改善しよう

import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.metrics import roc_auc_score, average_precision_score, f1_score, recall_score, confusion_matrix

# データ読み込みと前処理
df = pd.read_csv('WA_Fn-UseC_-Telco-Customer-Churn.csv')
df['TotalCharges'] = pd.to_numeric(df['TotalCharges'], errors='coerce').fillna(0)
df = df.drop('customerID', axis=1)
df['Churn'] = (df['Churn'] == 'Yes').astype(int)

# エンコーディング
binary_cols = ['gender', 'Partner', 'Dependents', 'PhoneService', 'PaperlessBilling']
le = LabelEncoder()
for col in binary_cols:
    df[col] = le.fit_transform(df[col])

service_cols = ['MultipleLines', 'OnlineSecurity', 'OnlineBackup',
                'DeviceProtection', 'TechSupport', 'StreamingTV', 'StreamingMovies']
for col in service_cols:
    df[col] = df[col].replace({'No internet service': 'No', 'No phone service': 'No'})

multi_cols = service_cols + ['InternetService', 'Contract', 'PaymentMethod']
df = pd.get_dummies(df, columns=multi_cols, drop_first=True)

# 分割
X = df.drop('Churn', axis=1)
y = df['Churn']
X_train, X_temp, y_train, y_temp = train_test_split(X, y, test_size=0.4, random_state=42, stratify=y)
X_val, X_test, y_val, y_test = train_test_split(X_temp, y_temp, test_size=0.5, random_state=42, stratify=y_temp)

# スケーリング
numeric_cols = ['tenure', 'MonthlyCharges', 'TotalCharges']
scaler = StandardScaler()
X_train[numeric_cols] = scaler.fit_transform(X_train[numeric_cols])
X_val[numeric_cols] = scaler.transform(X_val[numeric_cols])
X_test[numeric_cols] = scaler.transform(X_test[numeric_cols])

# ベースラインモデル
baseline = LogisticRegression(max_iter=1000, class_weight='balanced', random_state=42)
baseline.fit(X_train, y_train)

y_proba = baseline.predict_proba(X_val)[:, 1]
y_pred = baseline.predict(X_val)

# 実験ログ
experiment_log = [{
    'Experiment': 'Baseline (LR)',
    'AUC-ROC': roc_auc_score(y_val, y_proba),
    'PR-AUC': average_precision_score(y_val, y_proba),
    'F1': f1_score(y_val, y_pred),
    'Recall': recall_score(y_val, y_pred),
}]
print(pd.DataFrame(experiment_log).to_string(index=False))
print(f"\n混同行列:\n{confusion_matrix(y_val, y_pred)}")

from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
import optuna

# Step A: モデル比較
models = {
    'Random Forest': RandomForestClassifier(n_estimators=100, class_weight='balanced', random_state=42),
    'XGBoost': XGBClassifier(n_estimators=100, scale_pos_weight=2.77, random_state=42, eval_metric='auc', use_label_encoder=False),
    'LightGBM': LGBMClassifier(n_estimators=100, is_unbalance=True, random_state=42, verbose=-1),
}

for name, model in models.items():
    model.fit(X_train, y_train)
    y_proba = model.predict_proba(X_val)[:, 1]
    y_pred = model.predict(X_val)
    experiment_log.append({
        'Experiment': f'Step A: {name}',
        'AUC-ROC': roc_auc_score(y_val, y_proba),
        'PR-AUC': average_precision_score(y_val, y_proba),
        'F1': f1_score(y_val, y_pred),
        'Recall': recall_score(y_val, y_pred),
    })

# Step B: 特徴量エンジニアリング（元データに対して実施後、再分割）
df_fe = pd.read_csv('WA_Fn-UseC_-Telco-Customer-Churn.csv')
df_fe['TotalCharges'] = pd.to_numeric(df_fe['TotalCharges'], errors='coerce').fillna(0)

# 新特徴量
df_fe['num_services'] = df_fe[['OnlineSecurity','OnlineBackup','DeviceProtection',
                                'TechSupport','StreamingTV','StreamingMovies']].apply(
    lambda row: (row=='Yes').sum(), axis=1)
df_fe['is_new_customer'] = (df_fe['tenure'] <= 6).astype(int)
df_fe['is_loyal'] = (df_fe['tenure'] >= 48).astype(int)
df_fe['charge_per_tenure'] = np.where(df_fe['tenure']>0, df_fe['MonthlyCharges']/df_fe['tenure'], df_fe['MonthlyCharges'])
df_fe['high_risk'] = ((df_fe['Contract']=='Month-to-month') & (df_fe['InternetService']=='Fiber optic') & (df_fe['tenure']<=12)).astype(int)
# ... 前処理と分割を実施 ...

# Step C: Optunaチューニング
def objective(trial):
    params = {
        'n_estimators': trial.suggest_int('n_estimators', 100, 500),
        'max_depth': trial.suggest_int('max_depth', 3, 10),
        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),
        'num_leaves': trial.suggest_int('num_leaves', 20, 150),
        'subsample': trial.suggest_float('subsample', 0.6, 1.0),
        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),
        'is_unbalance': True, 'random_state': 42, 'verbose': -1,
    }
    model = LGBMClassifier(**params)
    from sklearn.model_selection import cross_val_score, StratifiedKFold
    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
    return cross_val_score(model, X_train_fe, y_train, cv=cv, scoring='roc_auc').mean()

study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=50)
print(f"Best AUC-ROC (CV): {study.best_value:.4f}")

# 実験ログ出力
print(pd.DataFrame(experiment_log).to_string(index=False))

from sklearn.metrics import roc_curve, precision_recall_curve
import matplotlib.pyplot as plt

# 最終モデル
final_model = LGBMClassifier(**study.best_params, is_unbalance=True, random_state=42, verbose=-1)
final_model.fit(X_train_fe, y_train)

# テストデータ評価
y_test_proba = final_model.predict_proba(X_test_fe)[:, 1]
y_test_pred = final_model.predict(X_test_fe)

print("=== 最終モデル（テストデータ）===")
print(f"AUC-ROC: {roc_auc_score(y_test, y_test_proba):.4f}")
print(f"PR-AUC:  {average_precision_score(y_test, y_test_proba):.4f}")
print(f"F1:      {f1_score(y_test, y_test_pred):.4f}")
print(f"Recall:  {recall_score(y_test, y_test_pred):.4f}")

# ROC曲線
fpr, tpr, _ = roc_curve(y_test, y_test_proba)
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(fpr, tpr, 'b-', label=f'AUC={roc_auc_score(y_test, y_test_proba):.3f}')
plt.plot([0,1], [0,1], 'k--')
plt.xlabel('FPR'); plt.ylabel('TPR'); plt.title('ROC Curve'); plt.legend()

# PR曲線
precision, recall, _ = precision_recall_curve(y_test, y_test_proba)
plt.subplot(1, 2, 2)
plt.plot(recall, precision, 'r-', label=f'PR-AUC={average_precision_score(y_test, y_test_proba):.3f}')
plt.xlabel('Recall'); plt.ylabel('Precision'); plt.title('PR Curve'); plt.legend()
plt.tight_layout()
plt.savefig('final_model_curves.png', dpi=150)

# 特徴量重要度
fi = pd.DataFrame({'feature': X_train_fe.columns, 'importance': final_model.feature_importances_})
fi = fi.sort_values('importance', ascending=False)
print(f"\nTop 10 特徴量:\n{fi.head(10).to_string(index=False)}")

# 過学習チェック
train_auc = roc_auc_score(y_train, final_model.predict_proba(X_train_fe)[:, 1])
print(f"\n訓練AUC: {train_auc:.4f}, テストAUC: {roc_auc_score(y_test, y_test_proba):.4f}")
print(f"差分: {train_auc - roc_auc_score(y_test, y_test_proba):.4f}")

# 全実験の比較表
print(f"\n=== 実験比較 ===")
print(pd.DataFrame(experiment_log).to_string(index=False))