Bootcamp

# ===== Objectives =====
# 1. Load + chunk documents from a folder
# 2. Build a vector retriever
# 3. Ship a RAG chain with structured output
# 4. Add cheap evals + a CI regression gate

# ===== 0-5 min: scope =====
# Pick ONE folder (docs/, policies/, ./README.md + 10 markdown files).
# Decide the task: 'answer support questions using these docs'.
# Pick the model NOW (cost matters): gpt-4o-mini for the bootcamp.

# ===== 5-20 min: load + chunk =====
# pip install langchain langchain-openai langchain-chroma langchain-text-splitters
from langchain_community.document_loaders import DirectoryLoader, TextLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter

loader = DirectoryLoader('./docs', glob='**/*.md', loader_cls=TextLoader)
docs = loader.load()
print(f'loaded {len(docs)} docs')

splitter = RecursiveCharacterTextSplitter(chunk_size=900, chunk_overlap=120)
chunks = splitter.split_documents(docs)
print(f'split into {len(chunks)} chunks')

# ===== 20-30 min: vector store =====
from langchain_openai import OpenAIEmbeddings
from langchain_chroma import Chroma

vs = Chroma.from_documents(
    documents=chunks,
    embedding=OpenAIEmbeddings(model='text-embedding-3-small'),
    persist_directory='./vector-store',
)

retriever = vs.as_retriever(search_kwargs={'k': 4})

# ===== 30-45 min: RAG chain with structured output =====
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough
from pydantic import BaseModel, Field
from typing import List

class Answer(BaseModel):
    summary: str = Field(max_length=400)
    citations: List[str] = Field(default_factory=list)
    confident: bool

llm = ChatOpenAI(model='gpt-4o-mini', temperature=0)

prompt = ChatPromptTemplate.from_messages([
    ('system',
     'You answer questions using ONLY the provided context. Cite the source for every claim. '
     'If the context does not contain the answer, set confident=false and say so.'),
    ('human',
     'Question: {question}\n\nContext:\n{context}'),
])

def format_docs(docs):
    return '\n\n'.join(f'[doc {i+1}: {d.metadata.get("source","?")}]\n{d.page_content[:1200]}'
                          for i, d in enumerate(docs))

structured_llm = llm.with_structured_output(Answer)

rag = (
    { 'context': retriever | format_docs, 'question': RunnablePassthrough() }
    | prompt
    | structured_llm
)

print(rag.invoke('What is our refund policy for in-store purchases?'))

# ===== 45-55 min: cheap evals =====
golden = [
    ('How long do refunds take?', ['5 business days', 'within 5 days']),
    ('What is the warranty period?', ['12 months', '1 year']),
    ('Do you ship internationally?', ['no', 'AU only']),
]

def eval_contains_any(answer, expected_substrings):
    a = answer.summary.lower()
    return any(s.lower() in a for s in expected_substrings)

results = []
for q, expected in golden:
    a = rag.invoke(q)
    results.append({
        'q': q,
        'pass': eval_contains_any(a, expected),
        'confident': a.confident,
        'citations': len(a.citations),
        'answer': a.summary,
    })

print('pass rate:', sum(1 for r in results if r['pass']) / len(results))

# ===== 55-60 min: CI regression gate =====
# Save the golden + results to a JSON; load on every PR.
# Fail CI if pass rate drops more than N%.
#
# import json
# baseline = json.load(open('rag-baseline.json'))
# current = json.load(open('rag-current.json'))
# regression = baseline['pass_rate'] - current['pass_rate']
# if regression > 0.05:
#     raise SystemExit('regression: -' + str(regression))

# ===== Post-bootcamp =====
# - Add caching: from langchain.cache import InMemoryCache; set_llm_cache(...)
# - Add a router so cheap queries skip retrieval
# - Add an LLM-as-judge evaluator for the nuance the contains check misses
# - Run a small smoke set in CI; the full set nightly to manage cost
# - Add a per-user rate limiter and a cost ceiling

# ===== Pitfalls =====
# - Chunk size too big -> model ignores middle of context
# - Chunk size too small -> retrieval misses
# - No structured output -> downstream parsing is unreliable
# - No eval -> prompt changes regress silently
# - Public input flowing into system prompt -> prompt injection

# 30-day LangChain bootcamp in the lesson body.