Missing Data

import numpy as np
import pandas as pd

# 1) Detect
df = pd.DataFrame({
    'name':  ['Ada', 'Bo', None, 'Di'],
    'age':   [32, np.nan, 41, 22],
    'score': [85, 90, np.nan, np.nan],
    'city':  ['Sydney', 'Sydney', None, 'Brisbane'],
})

df.isna()                       # boolean DataFrame
df.isna().sum()                 # per column
df.isna().sum(axis=1)           # per row
df.isna().any(axis=1)           # rows with any missing
df.notna().all(axis=1)          # complete rows

# Visualise — quick scan
# df.isna().heatmap(...)  # matplotlib / seaborn (or missingno library)

# 2) Drop
df.dropna()                                  # rows with ANY missing
df.dropna(how='all')                         # rows ALL missing
df.dropna(subset=['age', 'city'])           # only consider some cols
df.dropna(axis=1)                            # drop columns instead of rows
df.dropna(thresh=3)                          # keep rows with >= 3 non-NA values

# 3) Fill — constant
df.fillna(0)
df.fillna({'age': 0, 'score': 0, 'city': 'Unknown', 'name': 'Anonymous'})

# Fill in place
df.fillna({'age': df['age'].median()}, inplace=True)

# 4) Fill — forward / backward (great for time series)
df['age'].ffill()                            # propagate last valid value forward
df['age'].bfill()                            # propagate next valid backward
df['age'].ffill(limit=1)                     # cap propagation

# 5) Fill — interpolate
s = pd.Series([1, np.nan, np.nan, 4, 5])
s.interpolate()                              # [1, 2, 3, 4, 5] — linear
s.interpolate(method='time')                 # for time-indexed series
s.interpolate(method='polynomial', order=2)  # quadratic fit

# 6) Fill with statistics
df['age']   = df['age'].fillna(df['age'].median())
df['score'] = df['score'].fillna(df['score'].mean())
df['city']  = df['city'].fillna(df['city'].mode().iloc[0])

# 7) Group-aware fill (impute within a group)
df['age'] = df.groupby('city')['age'].transform(lambda s: s.fillna(s.median()))

# 8) Sentinel values masquerading as missing
# e.g. -999 or 'N/A' from a CSV — coerce them first
df.replace({'age': -999}, np.nan, inplace=True)
df = pd.read_csv('data.csv', na_values=['', 'N/A', '?', '-999'])

# 9) Type-aware missing — nullable dtypes (Pandas 1.0+)
import pandas as pd
s = pd.array([1, 2, None], dtype='Int32')          # nullable integer
b = pd.array([True, None, False], dtype='boolean')  # nullable boolean
t = pd.array(['a', None, 'c'], dtype='string')      # nullable string
# These use pd.NA instead of np.nan — cleaner type semantics.

# 10) sklearn — impute as part of a pipeline
from sklearn.impute import SimpleImputer, KNNImputer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier

pipe = Pipeline([
    ('impute', SimpleImputer(strategy='median')),
    ('scale',  StandardScaler()),
    ('clf',    RandomForestClassifier()),
])
pipe.fit(X_train, y_train)
# The impute parameters are LEARNED on train and APPLIED on test — no leakage.

# More sophisticated:
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
imp = IterativeImputer(random_state=42)
X_imp = imp.fit_transform(X)

# 11) Don't silently lose data
# `df['x'] + df['y']` propagates NaN. Sum / mean SKIP NaN by default; corr ignores rows with NaN.
# Reduce surprise: explicitly choose drop or fill BEFORE the downstream op.

# 12) When NaN actually means something — make it explicit
# Sometimes 'missing' carries information (the user didn't answer).
# Add an indicator column:
df['age_missing'] = df['age'].isna().astype(int)
df['age'] = df['age'].fillna(df['age'].median())

# 13) Time-series specific
# Resample first, then ffill within a tolerance — common for irregular timestamps:
ts = ts.resample('1H').mean().ffill(limit=2)

# 14) Tools
# - missingno    : visualisation (matrix, bars, heatmap)
# - pandas-profiling / ydata-profiling: profile reports include missing-data summaries
# - great_expectations: assert missingness rates in data pipelines

import pandas as pd
df.isna().sum()              # count missing per column
df = df.dropna(subset=['email'])
df['age'] = df['age'].fillna(df['age'].median())