PCA

from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline
from sklearn.datasets import load_iris, fetch_olivetti_faces
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# 1) Tiny example — iris
iris = load_iris(as_frame=True)
X = iris.data
y = iris.target

# 2) ALWAYS scale before PCA — variance is scale-dependent
pipe = make_pipeline(StandardScaler(), PCA(n_components=2, random_state=42))
X2 = pipe.fit_transform(X)
print(X2.shape)                            # (150, 2)

# 3) Inspect explained variance
pca = pipe.named_steps['pca']
print(pca.explained_variance_ratio_)        # e.g. [0.7296, 0.2285]
print(pca.explained_variance_ratio_.sum())  # ~0.96 — almost all

# 4) Pick K via cumulative explained variance
scan = make_pipeline(StandardScaler(), PCA().fit(X))
cum = np.cumsum(scan.named_steps['pca'].explained_variance_ratio_)
plt.plot(range(1, len(cum) + 1), cum, 'o-')
plt.axhline(0.95, ls='--')
plt.xlabel('components'); plt.ylabel('cumulative variance')
plt.title('Scree-style plot — pick smallest K with ≥ 0.95')

K = np.searchsorted(cum, 0.95) + 1
print(f'using {K} components')

# Or let sklearn pick by variance threshold:
pca95 = PCA(n_components=0.95).fit(X)
print(pca95.n_components_)

# 5) Inverse transform — reconstruction
X_rec = pca.inverse_transform(X2)           # back to original feature space (with loss)
rec_err = np.mean((X - pca.inverse_transform(pipe.named_steps['pca'].transform(StandardScaler().fit_transform(X)))) ** 2)

# 6) Plot 2-D scatter
plt.scatter(X2[:, 0], X2[:, 1], c=y, cmap='tab10', s=30, alpha=0.8)
plt.xlabel('PC1'); plt.ylabel('PC2')
plt.title('Iris in PCA space')

# 7) Loadings — what each PC is made of
load = pd.DataFrame(
    pca.components_,
    columns=iris.feature_names,
    index=[f'PC{i+1}' for i in range(pca.n_components_)],
)
print(load.round(3))
# Big absolute values reveal which original features dominate each PC.

# 8) Biplot — scatter + feature arrows
def biplot(scores, components, feature_names):
    plt.scatter(scores[:, 0], scores[:, 1], c=y, cmap='tab10', s=30, alpha=0.7)
    for i, name in enumerate(feature_names):
        plt.arrow(0, 0, components[0, i] * 3, components[1, i] * 3, color='black')
        plt.text(components[0, i] * 3.2, components[1, i] * 3.2, name, color='red')
biplot(X2, pca.components_, iris.feature_names)

# 9) PCA as denoising — Olivetti faces
faces = fetch_olivetti_faces().images       # (400, 64, 64)
flat = faces.reshape(400, -1)
pca = PCA(n_components=50, whiten=False).fit(flat)
rec = pca.inverse_transform(pca.transform(flat))

fig, axes = plt.subplots(2, 5, figsize=(10, 4))
for i, ax in enumerate(axes.flat):
    img = (flat[i] if i < 5 else rec[i - 5]).reshape(64, 64)
    ax.imshow(img, cmap='gray'); ax.axis('off')

# Components ARE patterns — first few are large-scale variation; later ones are texture/noise.

# 10) Whitening — useful before some downstream models
PCA(n_components=10, whiten=True).fit_transform(X_scaled)
# Outputs have unit variance per component, decorrelated. Good for many downstream algos.

# 11) Incremental PCA — for data that doesn't fit in memory
from sklearn.decomposition import IncrementalPCA
ipca = IncrementalPCA(n_components=50)
for batch in chunked(reader, 1024):
    ipca.partial_fit(StandardScaler().fit_transform(batch))

# 12) Kernel PCA — non-linear dimensionality reduction
from sklearn.decomposition import KernelPCA
kpca = KernelPCA(n_components=2, kernel='rbf', gamma=0.5)
Xk = kpca.fit_transform(X)

# Try when PCA produces a sausage-shaped projection that doesn't reveal class structure.
# Consider UMAP / t-SNE for VISUALISATION (not for downstream models).

# 13) When PCA is the WRONG tool
#   • Categorical features → one-hot + PCA leaks structure; consider MCA / PLS
#   • Very high-dimensional sparse text → TruncatedSVD (faster, doesn't densify)
#   • Non-linear manifolds → KernelPCA, UMAP, IsoMap, t-SNE
#   • Class separation is the goal, not variance → LDA (supervised)
#   • You need interpretable features → don't use PCA; ICA / NMF may help

# 14) Common pitfalls
#   • Skipping scaling → high-variance features dominate
#   • Fitting PCA on the TEST set leaks information — fit on train, transform both
#   • Treating PCA as feature selection — it makes COMBINATIONS, you can't drop a column directly
#   • Reading too much into a single PC's direction — multiplied by -1 is the same component
#   • Using sklearn PCA on sparse matrices → use TruncatedSVD instead
#   • PCA before tree models (Random Forest, GBT) — usually unnecessary and harmful for interpretability

# 15) Quick recipe — when to reach for PCA
#   ✓ Visualise high-D data in 2-D / 3-D
#   ✓ Decorrelate features before a model that assumes independence
#   ✓ Compress for storage or speed (e.g. embeddings → 50-D)
#   ✓ Denoise structured data (faces, signals)
#   ✗ Class separation as the primary goal — use LDA or a classifier directly

from sklearn.decomposition import PCA
pca = PCA(n_components=2).fit(Xtr)
Xtr_2d = pca.transform(Xtr)
print('explained variance:', pca.explained_variance_ratio_)