RAG Intro

from langchain_anthropic import ChatAnthropic
from langchain_community.document_loaders import WebBaseLoader, DirectoryLoader, TextLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain_openai import OpenAIEmbeddings
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnablePassthrough
from langchain_core.output_parsers import StrOutputParser

# 1) Load documents
loader = DirectoryLoader('./docs', glob='**/*.md', loader_cls=TextLoader)
docs = loader.load()
print(f'Loaded {len(docs)} documents')

# 2) Chunk — overlap stops semantic boundaries from cutting concepts in half
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    length_function=len,
    separators=['\n\n', '\n', '. ', ' ', ''],
)
chunks = splitter.split_documents(docs)
print(f'Created {len(chunks)} chunks')

# 3) Embed + store in a vector DB
embeddings = OpenAIEmbeddings(model='text-embedding-3-small')
vs = Chroma.from_documents(chunks, embeddings, persist_directory='./chroma')
vs.persist()

# 4) Build a retriever
retriever = vs.as_retriever(
    search_type='similarity',
    search_kwargs={'k': 4},
)

# 5) Prompt — include retrieved context + the question
prompt = ChatPromptTemplate.from_template('''Answer the question using only the context below.
If the context doesn't contain the answer, say "I don't know based on the provided documents."

Context:
{context}

Question: {question}

Answer:''')

# 6) LCEL chain — retrieve, format, prompt, model, parse
llm = ChatAnthropic(model='claude-opus-4-7', temperature=0)

def format_docs(docs):
    return '\n\n---\n\n'.join(d.page_content for d in docs)

chain = (
    {'context': retriever | format_docs, 'question': RunnablePassthrough()}
    | prompt
    | llm
    | StrOutputParser()
)

# 7) Ask
answer = chain.invoke('What is the company refund policy?')
print(answer)

# 8) Stream tokens
for chunk in chain.stream('Summarise our SLA in 3 bullets.'):
    print(chunk, end='', flush=True)

# 9) Cite sources — return docs alongside the answer
from langchain.chains import RetrievalQAWithSourcesChain
qa = RetrievalQAWithSourcesChain.from_llm(llm, retriever=retriever)
result = qa.invoke({'question': 'What is our PTO policy?'})
print(result['answer'])
print('Sources:', result['sources'])

# 10) Improve quality (rough order of impact)
#   - Better chunking: semantic boundaries (headings) over fixed-size
#   - Larger embedding model (text-embedding-3-large)
#   - Hybrid retrieval: vector + keyword (BM25)
#   - Reranking: cross-encoder over top-50 results
#   - Query rewriting: have the LLM expand the question first
#   - Larger k + smarter prompt formatting
#   - Eval set + measure (recall@k, faithfulness, answer quality)

# 11) Production checklist
#   • Persistent vector DB (Pinecone, Qdrant, pgvector) — not in-process
#   • Background re-indexing as docs change
#   • Source attribution in the UI — users distrust un-cited answers
#   • Guardrails: refuse out-of-scope questions; don't generate beyond the context

# RAG = Retrieval-Augmented Generation.
# Embed your docs → store in a vector DB → at query time, retrieve top-k → feed to the LLM as context.