演習：A/Bテストの結果を分析しよう

# === 基本統計量 ===
summary = ab_data.groupby('group').agg(
    n_users=('user_id', 'count'),
    n_purchased=('purchased', 'sum'),
    cvr=('purchased', 'mean'),
    avg_amount_all=('amount', 'mean'),
    avg_pages=('pages_viewed', 'mean'),
    avg_time=('time_on_site', 'mean')
).round(4)
print("=== 基本統計量 ===")
print(summary)

# 購入者のみの平均金額
buyers = ab_data[ab_data['purchased'] == 1]
buyer_stats = buyers.groupby('group')['amount'].agg(['mean', 'median', 'std']).round(0)
print("\n=== 購入者の注文金額 ===")
print(buyer_stats)

# === デバイス別CVR ===
device_cvr = ab_data.pivot_table(
    values='purchased', index='device', columns='group', aggfunc='mean'
).round(4)
print("\n=== デバイス別CVR ===")
print(device_cvr)

# === 年齢層別CVR ===
age_cvr = ab_data.pivot_table(
    values='purchased', index='age_group', columns='group', aggfunc='mean'
).round(4)
print("\n=== 年齢層別CVR ===")
print(age_cvr)

# === 属性の均等性チェック ===
print("\n=== デバイス分布 ===")
device_dist = pd.crosstab(ab_data['group'], ab_data['device'], normalize='index').round(3)
print(device_dist)

print("\n=== 年齢層分布 ===")
age_dist = pd.crosstab(ab_data['group'], ab_data['age_group'], normalize='index').round(3)
print(age_dist)

# === 1. CVRの検定（カイ二乗検定）===
contingency = pd.crosstab(ab_data['group'], ab_data['purchased'])
chi2, p_value_cvr, dof, expected = stats.chi2_contingency(contingency)
print("=== CVRの検定（カイ二乗検定）===")
print(f"カイ二乗統計量: {chi2:.4f}")
print(f"p値: {p_value_cvr:.4f}")

# CVRの差と信頼区間
cvr_control = control['purchased'].mean()
cvr_test = test['purchased'].mean()
cvr_diff = cvr_test - cvr_control

# CVR差の95%信頼区間
se_diff = np.sqrt(
    cvr_control * (1 - cvr_control) / n_control +
    cvr_test * (1 - cvr_test) / n_test
)
ci_lower = cvr_diff - 1.96 * se_diff
ci_upper = cvr_diff + 1.96 * se_diff
print(f"\nCVR差: {cvr_diff:.4f} ({cvr_diff/cvr_control*100:.1f}%改善)")
print(f"95%CI: [{ci_lower:.4f}, {ci_upper:.4f}]")

# === 2. 購入金額の検定（Welchのt検定）===
buyers_control = control[control['purchased'] == 1]['amount']
buyers_test = test[test['purchased'] == 1]['amount']

t_stat, p_value_amount = stats.ttest_ind(buyers_control, buyers_test, equal_var=False)
print("\n=== 購入金額の検定（Welchのt検定）===")
print(f"t統計量: {t_stat:.4f}")
print(f"p値: {p_value_amount:.4f}")

# === 3. 効果量 ===
# Cohen's d for amount
pooled_std = np.sqrt(
    ((len(buyers_control)-1)*buyers_control.var() + (len(buyers_test)-1)*buyers_test.var()) /
    (len(buyers_control) + len(buyers_test) - 2)
)
cohens_d = (buyers_test.mean() - buyers_control.mean()) / pooled_std
print(f"\nCohen's d (金額): {cohens_d:.3f}")

# Odds Ratio for CVR
odds_control = cvr_control / (1 - cvr_control)
odds_test = cvr_test / (1 - cvr_test)
odds_ratio = odds_test / odds_control
print(f"オッズ比 (CVR): {odds_ratio:.3f}")

# === 4. デバイス別サブグループ分析 ===
print("\n=== デバイス別サブグループ分析 ===")
for device in ['desktop', 'mobile', 'tablet']:
    sub = ab_data[ab_data['device'] == device]
    sub_contingency = pd.crosstab(sub['group'], sub['purchased'])
    chi2_sub, p_sub, _, _ = stats.chi2_contingency(sub_contingency)
    cvr_c = sub[sub['group'] == 'control']['purchased'].mean()
    cvr_t = sub[sub['group'] == 'test']['purchased'].mean()
    print(f"{device}: control={cvr_c:.4f}, test={cvr_t:.4f}, diff={cvr_t-cvr_c:.4f}, p={p_sub:.4f}")

# === ビジネスインパクト算出 ===
monthly_visitors = 500000  # 月間訪問数（仮定）

# 現行の月間売上
current_cvr = cvr_control
current_avg_amount = buyers_control.mean()
current_monthly_revenue = monthly_visitors * current_cvr * current_avg_amount

# 新デザインの月間売上
new_cvr = cvr_test
new_avg_amount = buyers_test.mean()
new_monthly_revenue = monthly_visitors * new_cvr * new_avg_amount

# 増加額
revenue_increase = new_monthly_revenue - current_monthly_revenue
revenue_increase_pct = revenue_increase / current_monthly_revenue * 100

print("=== ビジネスインパクト ===")
print(f"現行月間売上推定: {current_monthly_revenue:,.0f}円")
print(f"新デザイン月間売上推定: {new_monthly_revenue:,.0f}円")
print(f"増加額: {revenue_increase:,.0f}円 ({revenue_increase_pct:.1f}%)")
print(f"年間換算: {revenue_increase * 12:,.0f}円")

# 信頼区間を用いたシナリオ
worst_cvr_diff = ci_lower
best_cvr_diff = ci_upper

worst_revenue = monthly_visitors * (current_cvr + worst_cvr_diff) * new_avg_amount - current_monthly_revenue
best_revenue = monthly_visitors * (current_cvr + best_cvr_diff) * new_avg_amount - current_monthly_revenue

print(f"\n=== シナリオ分析 ===")
print(f"悲観シナリオ: 月間 {worst_revenue:,.0f}円")
print(f"基本シナリオ: 月間 {revenue_increase:,.0f}円")
print(f"楽観シナリオ: 月間 {best_revenue:,.0f}円")

# === 結論レポート ===
print("\n" + "=" * 60)
print("A/Bテスト分析レポート - 結論")
print("=" * 60)
print(f"""
【結論】新デザインの導入を推奨する

【根拠】
1. CVR: {cvr_control:.3%} → {cvr_test:.3%} (p={p_value_cvr:.4f}, 有意)
2. 月間売上増加見込み: {revenue_increase:,.0f}円
3. 悲観シナリオでも{worst_revenue:,.0f}円の増加

【留意事項】
- テスト期間が2週間と短い（季節変動の影響を受ける可能性）
- デバイス別の効果差を考慮し、段階的な展開を推奨
- 導入後もモニタリングを継続し、長期的な効果を検証すること
- 新奇性効果（テスト初期だけ高い可能性）に注意
""")