演習：推薦アルゴリズムを実装・比較しよう

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from surprise import Dataset

# MovieLens 100Kデータの読み込み
data = Dataset.load_builtin('ml-100k')
raw_ratings = data.raw_ratings  # (user, item, rating, timestamp)

df = pd.DataFrame(raw_ratings, columns=['user_id', 'item_id', 'rating', 'timestamp'])
df['rating'] = df['rating'].astype(float)

# 基本統計量
n_users = df['user_id'].nunique()
n_items = df['item_id'].nunique()
n_ratings = len(df)
sparsity = 1 - n_ratings / (n_users * n_items)

print(f"ユーザー数: {n_users}")
print(f"アイテム数: {n_items}")
print(f"評価数: {n_ratings}")
print(f"スパース率: {sparsity:.4f} ({sparsity*100:.1f}%)")
# ユーザー数: 943, アイテム数: 1682, 評価数: 100000, スパース率: 93.7%

# 評価分布
fig, axes = plt.subplots(1, 3, figsize=(15, 4))

df['rating'].hist(bins=5, ax=axes[0])
axes[0].set_title('評価値の分布')
axes[0].set_xlabel('Rating')

df.groupby('user_id').size().hist(bins=50, ax=axes[1])
axes[1].set_title('ユーザーごとの評価件数')
axes[1].set_xlabel('Number of ratings')

df.groupby('item_id').size().hist(bins=50, ax=axes[2])
axes[2].set_title('アイテムごとの被評価件数（ロングテール）')
axes[2].set_xlabel('Number of ratings')

plt.tight_layout()
plt.show()

from surprise import KNNBasic, KNNWithMeans
from surprise.model_selection import cross_validate

reader = Reader(rating_scale=(1, 5))
dataset = Dataset.load_from_df(df[['user_id', 'item_id', 'rating']], reader)

# User-Based CF
user_cf = KNNWithMeans(k=20, sim_options={
    'name': 'cosine', 'user_based': True
})
user_results = cross_validate(user_cf, dataset, measures=['RMSE', 'MAE'], cv=5)

# Item-Based CF
item_cf = KNNWithMeans(k=20, sim_options={
    'name': 'pearson_baseline', 'user_based': False
})
item_results = cross_validate(item_cf, dataset, measures=['RMSE', 'MAE'], cv=5)

print(f"User-Based CF - RMSE: {user_results['test_rmse'].mean():.4f}, "
      f"MAE: {user_results['test_mae'].mean():.4f}")
print(f"Item-Based CF - RMSE: {item_results['test_rmse'].mean():.4f}, "
      f"MAE: {item_results['test_mae'].mean():.4f}")

# k の最適化
k_values = [5, 10, 20, 40, 80]
results_by_k = []

for k in k_values:
    algo = KNNWithMeans(k=k, sim_options={
        'name': 'pearson_baseline', 'user_based': False
    })
    cv = cross_validate(algo, dataset, measures=['RMSE'], cv=5, verbose=False)
    results_by_k.append({
        'k': k,
        'RMSE': cv['test_rmse'].mean()
    })

results_df = pd.DataFrame(results_by_k)
print(results_df)
# 通常k=20〜40あたりが最適

from surprise import SVD, NMF

# SVD
svd = SVD(n_factors=50, n_epochs=20, lr_all=0.005, reg_all=0.02)
svd_results = cross_validate(svd, dataset, measures=['RMSE', 'MAE'], cv=5)

# NMF
nmf = NMF(n_factors=50, n_epochs=50)
nmf_results = cross_validate(nmf, dataset, measures=['RMSE', 'MAE'], cv=5)

print(f"SVD - RMSE: {svd_results['test_rmse'].mean():.4f}")
print(f"NMF - RMSE: {nmf_results['test_rmse'].mean():.4f}")

# 潜在因子数の最適化
factor_values = [10, 30, 50, 100, 200]
svd_by_factors = []

for f in factor_values:
    algo = SVD(n_factors=f, n_epochs=20)
    cv = cross_validate(algo, dataset, measures=['RMSE'], cv=5, verbose=False)
    svd_by_factors.append({
        'factors': f,
        'RMSE': cv['test_rmse'].mean()
    })

# 全手法比較
comparison = pd.DataFrame([
    {'手法': 'User-Based CF', 'RMSE': user_results['test_rmse'].mean()},
    {'手法': 'Item-Based CF', 'RMSE': item_results['test_rmse'].mean()},
    {'手法': 'SVD', 'RMSE': svd_results['test_rmse'].mean()},
    {'手法': 'NMF', 'RMSE': nmf_results['test_rmse'].mean()},
])
print("\n=== 全手法比較 ===")
print(comparison.sort_values('RMSE'))
# 通常SVDが最も良い性能を示す

from surprise.model_selection import train_test_split
from collections import defaultdict

def get_top_n(predictions, n=10):
    """予測結果からTop-N推薦を生成"""
    top_n = defaultdict(list)
    for uid, iid, true_r, est, _ in predictions:
        top_n[uid].append((iid, est))
    for uid, ratings in top_n.items():
        ratings.sort(key=lambda x: x[1], reverse=True)
        top_n[uid] = ratings[:n]
    return top_n

def precision_recall_at_k(predictions, k=10, threshold=4.0):
    """Precision@KとRecall@Kを計算"""
    user_est_true = defaultdict(list)
    for uid, _, true_r, est, _ in predictions:
        user_est_true[uid].append((est, true_r))

    precisions = {}
    recalls = {}

    for uid, user_ratings in user_est_true.items():
        user_ratings.sort(key=lambda x: x[0], reverse=True)
        top_k = user_ratings[:k]

        n_relevant = sum(1 for _, true_r in user_ratings if true_r >= threshold)
        n_rec_relevant = sum(1 for est, true_r in top_k if true_r >= threshold)

        precisions[uid] = n_rec_relevant / k if k > 0 else 0
        recalls[uid] = n_rec_relevant / n_relevant if n_relevant > 0 else 0

    return (
        sum(precisions.values()) / len(precisions),
        sum(recalls.values()) / len(recalls)
    )

# 各手法でTop-N評価
trainset, testset = train_test_split(dataset, test_size=0.2, random_state=42)

models = {
    'Item-Based CF': KNNWithMeans(k=20, sim_options={
        'name': 'pearson_baseline', 'user_based': False
    }),
    'SVD': SVD(n_factors=50, n_epochs=20),
    'NMF': NMF(n_factors=50),
}

for name, model in models.items():
    model.fit(trainset)
    predictions = model.test(testset)
    precision, recall = precision_recall_at_k(predictions, k=10, threshold=4.0)
    print(f"{name} - Precision@10: {precision:.4f}, Recall@10: {recall:.4f}")

# NetShop社への提案
# ECサイトでは暗黙的フィードバック（購買/閲覧）が主 → ALS推奨
# 新商品も多い → コンテンツベースとのハイブリッドが必要

NetShop社への提案:

手法比較（精度以外含む）:
| 手法 | RMSE | Cold Start | 計算コスト | 更新性 |
|------|------|-----------|----------|--------|
| User-CF | 0.97 | 弱い | 高 | 遅い |
| Item-CF | 0.94 | 弱い | 中 | 安定 |
| SVD | 0.93 | 弱い | 低 | 要再学習 |
| コンテンツ | N/A | 強い | 低 | 即時 |

推奨: ハイブリッドアプローチ
  Phase 1: Item-CF + 人気ランキング（即効性）
  Phase 2: SVD/ALS + コンテンツベース（精度向上）
  Phase 3: セッションベース + リアルタイム化（体験向上）