sort / uniq

# ===== sort: default is lex, ascending =====
sort names.txt
sort -r names.txt              # reverse
sort -f names.txt              # case-insensitive
sort -u names.txt              # unique on the whole line (no extra uniq needed)

# Numeric:
sort -n nums.txt
sort -h sizes.txt              # human numeric: 1K, 2M, 3G

# By key (column):
sort -t, -k2,2 -n sales.csv    # sort csv by 2nd column numerically
sort -t: -k3,3n -k1,1 /etc/passwd  # multi-key

# Stable:
sort -s -k1,1 file             # preserve original order for equal keys

# ===== uniq: needs a SORTED input =====
sort access.log | uniq           # collapse adjacent duplicates
sort access.log | uniq -c        # count occurrences
sort access.log | uniq -d        # only duplicated lines
sort access.log | uniq -u        # only unique (appears once)

# uniq -c output looks like:
#   17 GET /api/users
#    9 POST /api/login

# ===== The 'top N' one-liner =====
awk '{print $1}' access.log | sort | uniq -c | sort -rn | head -10
# 1. Pick column 1 (IP)
# 2. Sort to group duplicates
# 3. Count
# 4. Re-sort by count desc
# 5. Top 10

# ===== Counting unique vs total =====
wc -l file                     # total lines
sort -u file | wc -l           # unique lines

# ===== Diffing two files for set membership =====
sort a.txt > a.s; sort b.txt > b.s
comm -23 a.s b.s               # in a only
comm -13 a.s b.s               # in b only
comm -12 a.s b.s               # in both

# Or with sort + uniq tricks:
sort a.txt b.txt | uniq -u     # appears in only one file
sort a.txt b.txt | uniq -d     # appears in both (once each)

# ===== Case-insensitive uniq =====
sort -f file | uniq -i

# ===== Group by key, then count =====
# Count requests per user from a log: 'user=alice path=/x ...'
grep -oE 'user=\S+' access.log | sort | uniq -c | sort -rn

# ===== With big files: parallelism =====
sort --parallel=4 -S 2G huge.csv > sorted.csv
# --parallel uses N threads; -S sets memory buffer (else uses TMPDIR for spills).

# ===== Patterns to internalise =====
# - sort | uniq -c | sort -rn is the universal 'top N' template
# - Always sort first; uniq is a streaming dedup, not a set
# - Sort numerically with -n; -h for sizes
# - Use comm on two sorted files for set algebra
# - For huge data, --parallel + -S, and a fast TMPDIR

# ===== Pitfalls =====
# - Running uniq without sort -> only collapses adjacent dupes
# - sort -n on mixed numeric/text -> mixed lines drift to the start; clean first
# - Lexical sort thinks '10' < '2' -> use -n
# - Files with CRLF endings -> uniq sees them as different; pipe through tr -d '\r' first
# - Locale-dependent sort: LANG=C sort is faster and stable for ASCII

sort file | uniq -c | sort -nr | head