Boolean Filtering

import pandas as pd

df = pd.DataFrame({
    'name':  ['Alex', 'Sam', 'Lee', 'Casey'],
    'age':   [30, 25, 40, 35],
    'city':  ['Sydney', 'Sydney', 'Melbourne', 'Perth'],
    'spent': [1200, 800, 450, 990],
})

# ===== Boolean masks =====
df[df.age > 28]
df[(df.age > 28) & (df.city == 'Sydney')]
df[~(df.city == 'Sydney')]          # NOT

# CAREFUL: use & | ~ (not 'and' / 'or' / 'not') with parentheses.

# ===== .loc with conditions =====
df.loc[df.age > 28, ['name', 'spent']]
df.loc[df.age > 28, 'spent'] = 0     # conditional assignment

# ===== .query (readable for ad-hoc) =====
df.query('age > 28 and city == "Sydney"')
df.query('age in [30, 40]')
df.query('city not in ["Perth"]')

# Reference a Python variable with @:
threshold = 28
df.query('age > @threshold')

# ===== isin =====
df[df.city.isin(['Sydney', 'Perth'])]
df[~df.city.isin(['Sydney'])]

# ===== between =====
df[df.age.between(25, 35)]               # inclusive
df[df.age.between(25, 35, inclusive='left')]

# ===== String matching =====
df[df.name.str.startswith('A')]
df[df.name.str.contains('a', case=False, na=False)]
df[df.name.str.match(r'^[A-C]')]
df[df.city.str.endswith('y')]

# ===== Null handling =====
df[df.spent.isna()]
df[df.spent.notna()]
df.dropna(subset=['spent'])
df.fillna({'spent': 0})

# ===== Multiple columns =====
mask = (df.age > 28) & (df.city == 'Sydney') & (df.spent > 1000)
df[mask]

# Build masks as variables for clarity in complex filters.

# ===== Sample / nlargest / nsmallest =====
df.sample(n=2)                          # random rows
df.sample(frac=0.5)                     # 50% random
df.nlargest(2, 'spent')
df.nsmallest(2, 'age')

# ===== Group + filter =====
# Keep groups whose mean spent > 700:
df.groupby('city').filter(lambda g: g.spent.mean() > 700)

# ===== Date filtering =====
df['date'] = pd.to_datetime(['2024-04-01','2024-04-15','2024-05-01','2024-05-15'])
df[df.date >= '2024-05-01']
df[df.date.dt.month == 4]
df[df.date.between('2024-04-10', '2024-05-10')]

# ===== Patterns to internalise =====
# - Use & | ~ with parentheses for combined masks
# - .query() for one-off readable filters
# - isin / between / str.contains for common shapes
# - Save masks to variables for complex filters

# ===== Pitfalls =====
# - 'and' / 'or' instead of '& | ~' -> truth value error
# - Forgetting parentheses around each clause
# - .str on a column with NaN without na=False -> NaN in mask
# - Chained assignment df[mask].col = ... -> SettingWithCopyWarning; use .loc

import pandas as pd
df = pd.read_csv('users.csv')
adults = df[(df['age'] >= 18) & (df['country'] == 'AU')]
print(adults.head())