LangSmith (eval & tracing)

# 1) Setup — environment variables
import os
os.environ['LANGCHAIN_TRACING_V2'] = 'true'
os.environ['LANGCHAIN_API_KEY']    = 'ls_...'
os.environ['LANGCHAIN_PROJECT']     = 'my-rag-app'
os.environ['LANGCHAIN_ENDPOINT']    = 'https://api.smith.langchain.com'

# 2) Trace anything LangChain runs — no extra code
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser

llm = ChatOpenAI(model='gpt-4o-mini')
chain = ChatPromptTemplate.from_template('Translate to French: {text}') | llm | StrOutputParser()

result = chain.invoke({'text': 'hello world'})
# Every LLM call, every prompt format, every retrieval step appears in LangSmith.
# Browse at https://smith.langchain.com -> your project.

# 3) Trace arbitrary Python code with @traceable
from langsmith import traceable

@traceable(run_type='llm', name='my-step')
def summarise(text: str) -> str:
    return llm.invoke(f'Summarise: {text}').content

@traceable(run_type='retriever', name='vector-search')
def search(query: str) -> list:
    # returns documents; LangSmith logs the query + result
    return vector_store.similarity_search(query, k=5)

summarise('long text here')
# Now even non-LangChain code traces into the same UI.

# 4) Add custom metadata + tags to a run
result = chain.invoke(
    {'text': 'hello'},
    config={'tags': ['user:42', 'environment:prod'], 'metadata': {'user_id': 42, 'session': 'abc'}},
)

# 5) Datasets — your eval set lives in LangSmith
from langsmith import Client
client = Client()

dataset = client.create_dataset(
    dataset_name='translation-eval',
    description='French translations of common phrases',
)

client.create_examples(
    inputs=[
        {'text': 'hello'},
        {'text': 'good morning'},
        {'text': 'thank you'},
    ],
    outputs=[
        {'expected': 'bonjour'},
        {'expected': 'bonjour'},
        {'expected': 'merci'},
    ],
    dataset_id=dataset.id,
)

# Promote successful production runs to the dataset:
# Click 'Add to Dataset' in the LangSmith UI on a run you want as a regression test.

# 6) Run evaluations against a dataset
from langsmith.evaluation import evaluate, LangChainStringEvaluator

def correctness(run, example) -> dict:
    pred = run.outputs.get('output', '')
    expected = example.outputs.get('expected', '').lower()
    return {'score': 1 if expected in pred.lower() else 0}

results = evaluate(
    lambda x: chain.invoke(x),                              # what to evaluate
    data='translation-eval',
    evaluators=[correctness, LangChainStringEvaluator('embedding_distance')],
    experiment_prefix='gpt-4o-mini-baseline',
)

# Compare experiments side-by-side; rerun on a new model to A/B test.

# 7) LLM-as-judge evaluator
from langchain_core.prompts import ChatPromptTemplate

JUDGE = ChatPromptTemplate.from_template('''
You are evaluating an answer.
Question: {question}
Answer: {answer}
Reference: {reference}
Grade between 0 and 1 (1 = perfect, 0 = incorrect). Reply with just the number.
''')
judge_chain = JUDGE | llm | StrOutputParser()

def llm_judge(run, example) -> dict:
    score = float(judge_chain.invoke({
        'question':  example.inputs['text'],
        'answer':    run.outputs['output'],
        'reference': example.outputs['expected'],
    }))
    return {'score': score, 'key': 'llm_judge'}

# Pass alongside structural evaluators — best of both worlds.

# 8) Prompt management — versions, hub, deploy
# Pull a prompt from the LangSmith hub:
from langchain import hub
prompt = hub.pull('rlm/rag-prompt')

# Push a prompt for your team:
hub.push('my-team/my-rag-prompt', prompt)

# Prompts have versions; load a specific one:
prompt = hub.pull('my-team/my-rag-prompt:e7e6c11d')

# 9) Feedback — capture thumbs-up/down from end users
from langsmith import Client
client = Client()

# After your chain runs, get the run id from the callback or response
run_id = result.run_id          # if you tagged it; or fetch from list_runs
client.create_feedback(run_id=run_id, key='user_rating', score=1, comment='Helpful!')

# Build feedback loops into product UI; LangSmith aggregates by user, session, version.

# 10) Filtering + sharing runs
# In the LangSmith UI:
# • Filter by tags (environment, user, feature flag)
# • Filter by metadata (user_id, session_id)
# • Filter by latency, token count, error
# • Save a 'view' for the team
# • Share a single run link for debugging

# 11) Programmatic queries
from langsmith import Client
client = Client()

for run in client.list_runs(project_name='my-rag-app', filter='eq(name, "qa")', limit=10):
    print(run.id, run.start_time, run.outputs)

# Use for: nightly quality checks, building dashboards, finding outliers.

# 12) Cost tracking
# LangSmith records token usage per run (when the LLM client reports it).
# Build dashboards: 'top 10 users by cost', 'cost per session', 'cost per chain version'.

# 13) Comparing experiments — A/B between models
# Run two evaluate() calls with different chains:
for model_name in ['gpt-4o-mini', 'gpt-4o']:
    llm = ChatOpenAI(model=model_name)
    chain = prompt | llm | StrOutputParser()
    evaluate(chain.invoke, data='translation-eval', experiment_prefix=model_name)
# Compare metrics side by side in the LangSmith UI.

# 14) Setup tips
# • Tag every run with environment ('prod', 'staging', 'dev')
# • Tag with user_id / session_id for support tickets ('show me runs for this customer')
# • Use one project per app (not per branch); use tags for variants
# • Promote production failures to your eval dataset — they're free test cases
# • For high-volume apps, sample traces (1-10%) to control cost

# 15) Common bugs
# • LANGCHAIN_TRACING_V2 not set → runs don't show up; check env
# • Traces missing for @traceable → set LANGCHAIN_TRACING_V2 + ensure langsmith is installed
# • Async tracing missing → use the latest LangSmith SDK; older versions had async gaps
# • Sensitive PII in traces → use 'hide_inputs' / 'hide_outputs' in @traceable or scrub before tracing
# • Evaluation feedback never showing → ensure feedback key + score are correct types
# • Prompt hub version drift → pin specific commit shas in production code
# • Dataset created in wrong project → projects + datasets are separate; check which workspace you're in

# Set env vars
export LANGSMITH_API_KEY=...
export LANGSMITH_TRACING=true
# Every chain run is now visible in smith.langchain.com — traces, eval, dataset comparisons.