모델 평가

# 필요한 라이브러리 import
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
    confusion_matrix, accuracy_score, precision_score, 
    recall_score, f1_score, roc_auc_score, roc_curve,
    precision_recall_curve, classification_report
)
from sklearn.preprocessing import StandardScaler

# 데이터 로드
data = load_breast_cancer()
X = pd.DataFrame(data.data, columns=data.feature_names)
y = data.target  # 0: 악성, 1: 양성

print(f"데이터 크기: {X.shape}")
print(f"클래스 분포:\n{pd.Series(y).value_counts()}")

# 데이터 분리
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# 데이터 스케일링
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 모델 학습
model = LogisticRegression(max_iter=1000, random_state=42)
model.fit(X_train_scaled, y_train)

# 예측
y_pred = model.predict(X_test_scaled)
y_pred_proba = model.predict_proba(X_test_scaled)[:, 1]  # 양성 클래스 확률

print("모델 학습 완료")

# Confusion Matrix 계산
cm = confusion_matrix(y_test, y_pred)
print("Confusion Matrix:")
print(cm)
print()

# TP, TN, FP, FN 추출
TN, FP, FN, TP = cm.ravel()
print(f"TP (True Positive): {TP}")
print(f"TN (True Negative): {TN}")
print(f"FP (False Positive): {FP}")
print(f"FN (False Negative): {FN}")
print()

# Confusion Matrix 시각화
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', 
            xticklabels=['Malignant', 'Benign'], 
            yticklabels=['Malignant', 'Benign'])
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.title('Confusion Matrix')
plt.show()

# 평가 지표 계산
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)
roc_auc = roc_auc_score(y_test, y_pred_proba)

print("=== 평가 지표 ===")
print(f"Accuracy:  {accuracy:.4f}")
print(f"Precision: {precision:.4f}")
print(f"Recall:    {recall:.4f}")
print(f"F1 Score:  {f1:.4f}")
print(f"ROC AUC:   {roc_auc:.4f}")
print()

# 공식으로 직접 계산하여 검증
print("=== 공식으로 직접 계산 ===")
print(f"Accuracy:  {(TP + TN) / (TP + TN + FP + FN):.4f}")
print(f"Precision: {TP / (TP + FP):.4f}")
print(f"Recall:    {TP / (TP + FN):.4f}")
print(f"F1 Score:  {2 * (precision * recall) / (precision + recall):.4f}")

# ROC 곡선 계산
fpr, tpr, thresholds_roc = roc_curve(y_test, y_pred_proba)

# ROC 곡선 시각화
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, label=f'ROC Curve (AUC = {roc_auc:.4f})')
plt.plot([0, 1], [0, 1], 'k--', label='Random Classifier')
plt.xlabel('False Positive Rate (FPR)')
plt.ylabel('True Positive Rate (TPR / Recall)')
plt.title('ROC Curve')
plt.legend()
plt.grid(True)
plt.show()

# 다양한 threshold에서 성능 계산
thresholds = np.arange(0.1, 1.0, 0.05)
precisions = []
recalls = []
f1_scores = []

for threshold in thresholds:
    y_pred_thresh = (y_pred_proba >= threshold).astype(int)
    precisions.append(precision_score(y_test, y_pred_thresh, zero_division=0))
    recalls.append(recall_score(y_test, y_pred_thresh))
    f1_scores.append(f1_score(y_test, y_pred_thresh))

# 시각화
plt.figure(figsize=(10, 6))
plt.plot(thresholds, precisions, label='Precision', marker='o')
plt.plot(thresholds, recalls, label='Recall', marker='s')
plt.plot(thresholds, f1_scores, label='F1 Score', marker='^')
plt.xlabel('Threshold')
plt.ylabel('Score')
plt.title('Performance Change by Threshold')
plt.legend()
plt.grid(True)
plt.axvline(x=0.5, color='r', linestyle='--', label='Default Threshold (0.5)')
plt.legend()
plt.show()

# 최적 threshold 찾기 (F1 Score 최대)
optimal_idx = np.argmax(f1_scores)
optimal_threshold = thresholds[optimal_idx]
print(f"최적 Threshold: {optimal_threshold:.2f}")
print(f"최적 F1 Score: {f1_scores[optimal_idx]:.4f}")

# Precision-Recall 곡선 계산
precision_curve, recall_curve, thresholds_pr = precision_recall_curve(y_test, y_pred_proba)

# Precision-Recall 곡선 시각화
plt.figure(figsize=(8, 6))
plt.plot(recall_curve, precision_curve)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.grid(True)
plt.show()

# Classification Report
print("=== Classification Report ===")
print(classification_report(y_test, y_pred, target_names=['Malignant', 'Benign']))

# Wine 데이터셋 로드 (다중분류)
from sklearn.datasets import load_wine

wine_data = load_wine()
X_wine = pd.DataFrame(wine_data.data, columns=wine_data.feature_names)
y_wine = wine_data.target  # 3개 클래스

print(f"데이터 크기: {X_wine.shape}")
print(f"클래스 분포:\n{pd.Series(y_wine).value_counts().sort_index()}")
print(f"클래스 이름: {wine_data.target_names}")

# 데이터 분리
X_train_wine, X_test_wine, y_train_wine, y_test_wine = train_test_split(
    X_wine, y_wine, test_size=0.2, random_state=42, stratify=y_wine
)

# 데이터 스케일링
scaler_wine = StandardScaler()
X_train_wine_scaled = scaler_wine.fit_transform(X_train_wine)
X_test_wine_scaled = scaler_wine.transform(X_test_wine)

# 모델 학습
model_wine = LogisticRegression(max_iter=1000, random_state=42)
model_wine.fit(X_train_wine_scaled, y_train_wine)

# 예측
y_pred_wine = model_wine.predict(X_test_wine_scaled)

print("모델 학습 완료")

# Confusion Matrix 계산 및 시각화
cm_wine = confusion_matrix(y_test_wine, y_pred_wine)

plt.figure(figsize=(10, 8))
sns.heatmap(cm_wine, annot=True, fmt='d', cmap='Blues',
            xticklabels=wine_data.target_names,
            yticklabels=wine_data.target_names)
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.title('Multi-class Confusion Matrix')
plt.show()

print("Confusion Matrix:")
print(cm_wine)

# 각 클래스별 Precision, Recall, F1 Score 계산
from sklearn.metrics import precision_recall_fscore_support

precision_per_class, recall_per_class, f1_per_class, support = precision_recall_fscore_support(
    y_test_wine, y_pred_wine, average=None
)

print("=== 클래스별 평가 지표 ===")
for i, class_name in enumerate(wine_data.target_names):
    print(f"\n{class_name}:")
    print(f"  Precision: {precision_per_class[i]:.4f}")
    print(f"  Recall:    {recall_per_class[i]:.4f}")
    print(f"  F1 Score:  {f1_per_class[i]:.4f}")
    print(f"  Support:   {support[i]}")

# Macro, Micro, Weighted 평균 계산
precision_macro = precision_score(y_test_wine, y_pred_wine, average='macro')
precision_micro = precision_score(y_test_wine, y_pred_wine, average='micro')
precision_weighted = precision_score(y_test_wine, y_pred_wine, average='weighted')

recall_macro = recall_score(y_test_wine, y_pred_wine, average='macro')
recall_micro = recall_score(y_test_wine, y_pred_wine, average='micro')
recall_weighted = recall_score(y_test_wine, y_pred_wine, average='weighted')

f1_macro = f1_score(y_test_wine, y_pred_wine, average='macro')
f1_micro = f1_score(y_test_wine, y_pred_wine, average='micro')
f1_weighted = f1_score(y_test_wine, y_pred_wine, average='weighted')

print("=== Average Metrics Comparison ===")
print(f"\n{'Metric':<15} {'Macro':<10} {'Micro':<10} {'Weighted':<10}")
print("-" * 45)
print(f"{'Precision':<15} {precision_macro:<10.4f} {precision_micro:<10.4f} {precision_weighted:<10.4f}")
print(f"{'Recall':<15} {recall_macro:<10.4f} {recall_micro:<10.4f} {recall_weighted:<10.4f}")
print(f"{'F1 Score':<15} {f1_macro:<10.4f} {f1_micro:<10.4f} {f1_weighted:<10.4f}")

# Classification Report
print("=== Classification Report ===")
print(classification_report(y_test_wine, y_pred_wine, 
                          target_names=wine_data.target_names))