演習：テキスト前処理パイプラインを構築しよう

import pandas as pd
import re
import matplotlib.pyplot as plt

# データの読み込み
train_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')

print(f"訓練データ: {train_df.shape}")
print(f"テストデータ: {test_df.shape}")
print(f"\nカラム: {train_df.columns.tolist()}")
print(f"\nラベル分布:\n{train_df['target'].value_counts()}")
print(f"\nサンプル:")
print(train_df[['text', 'target']].head(10))

# テキスト前処理
def clean_text(text):
    """テキスト前処理パイプライン"""
    # URL除去
    text = re.sub(r'https?://\S+|www\.\S+', '', text)
    # HTMLタグ除去
    text = re.sub(r'<.*?>', '', text)
    # メンション除去
    text = re.sub(r'@\w+', '', text)
    # ハッシュタグのシンボル除去（テキストは保持）
    text = re.sub(r'#(\w+)', r'\1', text)
    # 数字の除去
    text = re.sub(r'\d+', '', text)
    # 特殊文字除去
    text = re.sub(r'[^\w\s]', ' ', text)
    # 余分な空白除去
    text = re.sub(r'\s+', ' ', text).strip()
    # 小文字化
    text = text.lower()
    return text

train_df['clean_text'] = train_df['text'].apply(clean_text)

# 前処理前後の比較
print("\n前処理前後の比較:")
for i in range(5):
    print(f"  前: {train_df['text'].iloc[i]}")
    print(f"  後: {train_df['clean_text'].iloc[i]}")
    print()

# クラス分布の可視化
fig, ax = plt.subplots(figsize=(6, 4))
train_df['target'].value_counts().plot(kind='bar', ax=ax)
ax.set_title('Label Distribution')
ax.set_xlabel('Target')
ax.set_ylabel('Count')
plt.tight_layout()
plt.savefig('label_distribution.png')
plt.show()

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline
import numpy as np

# TF-IDFパイプライン
tfidf_pipeline = Pipeline([
    ('tfidf', TfidfVectorizer(
        max_features=10000,
        ngram_range=(1, 2),
        min_df=3,
        max_df=0.95,
        sublinear_tf=True,
    )),
    ('clf', LogisticRegression(max_iter=1000, C=1.0)),
])

X = train_df['clean_text']
y = train_df['target']

# 交差検証
scores = cross_val_score(tfidf_pipeline, X, y, cv=5, scoring='f1')
print(f"TF-IDF + Logistic Regression")
print(f"  F1スコア（5-fold CV）: {scores.mean():.4f} (+/- {scores.std():.4f})")
print(f"  各Fold: {scores}")

# ハイパーパラメータの比較
configs = [
    {"max_features": 5000, "ngram_range": (1, 1)},
    {"max_features": 10000, "ngram_range": (1, 2)},
    {"max_features": 20000, "ngram_range": (1, 3)},
]

for config in configs:
    pipe = Pipeline([
        ('tfidf', TfidfVectorizer(
            max_features=config['max_features'],
            ngram_range=config['ngram_range'],
            sublinear_tf=True,
        )),
        ('clf', LogisticRegression(max_iter=1000)),
    ])
    scores = cross_val_score(pipe, X, y, cv=5, scoring='f1')
    print(f"  max_features={config['max_features']}, "
          f"ngram={config['ngram_range']}: F1={scores.mean():.4f}")

from gensim.models import Word2Vec
from nltk.tokenize import word_tokenize
import nltk
nltk.download('punkt')

# トークン化
train_df['tokens'] = train_df['clean_text'].apply(word_tokenize)

# Word2Vecモデルの学習
w2v_model = Word2Vec(
    sentences=train_df['tokens'].tolist(),
    vector_size=200,
    window=5,
    min_count=2,
    workers=4,
    sg=1,
    epochs=20,
)

print(f"語彙サイズ: {len(w2v_model.wv)}")

# 文書ベクトルの作成
def get_document_vector(tokens, model, vector_size=200):
    """平均プーリングによる文書ベクトル"""
    vectors = [model.wv[t] for t in tokens if t in model.wv]
    if vectors:
        return np.mean(vectors, axis=0)
    return np.zeros(vector_size)

X_w2v = np.array([
    get_document_vector(tokens, w2v_model)
    for tokens in train_df['tokens']
])

print(f"文書ベクトルの形状: {X_w2v.shape}")

# 分類
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

w2v_pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('clf', LogisticRegression(max_iter=1000)),
])

scores_w2v = cross_val_score(w2v_pipeline, X_w2v, y, cv=5, scoring='f1')
print(f"\nWord2Vec + Logistic Regression")
print(f"  F1スコア（5-fold CV）: {scores_w2v.mean():.4f} (+/- {scores_w2v.std():.4f})")

from sentence_transformers import SentenceTransformer
import time

# Sentence Transformersによる埋め込み
st_model = SentenceTransformer('all-MiniLM-L6-v2')

start = time.time()
X_st = st_model.encode(
    train_df['clean_text'].tolist(),
    batch_size=64,
    show_progress_bar=True,
)
st_time = time.time() - start
print(f"Sentence Transformers埋め込み時間: {st_time:.2f}秒")
print(f"埋め込みの形状: {X_st.shape}")

# 分類
st_pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('clf', LogisticRegression(max_iter=1000)),
])

scores_st = cross_val_score(st_pipeline, X_st, y, cv=5, scoring='f1')
print(f"\nSentence Transformers + Logistic Regression")
print(f"  F1スコア（5-fold CV）: {scores_st.mean():.4f} (+/- {scores_st.std():.4f})")

# 3手法の比較
print("\n" + "="*60)
print("特徴抽出手法の比較")
print("="*60)
print(f"{'手法':<25} {'F1スコア':<15} {'次元数':<10}")
print("-"*60)
print(f"{'TF-IDF':<25} {scores.mean():.4f}        10000")
print(f"{'Word2Vec (平均)':<25} {scores_w2v.mean():.4f}        200")
print(f"{'Sentence Transformers':<25} {scores_st.mean():.4f}        384")
print("="*60)