Caching

from langchain_openai import ChatOpenAI
from langchain.globals import set_llm_cache

# 1) In-memory cache — quickest setup
from langchain_community.cache import InMemoryCache
set_llm_cache(InMemoryCache())

llm = ChatOpenAI(model='gpt-4o-mini')
# First call: hits the API
llm.invoke('What is the capital of France?')
# Second call (same prompt): cached, ~0 ms
llm.invoke('What is the capital of France?')

# Same chain semantics; just MUCH faster + cheaper for identical inputs.

# 2) SQLite cache — persistent
from langchain_community.cache import SQLiteCache
set_llm_cache(SQLiteCache(database_path='.langchain.db'))

# Across runs; great for dev + CI; portable.

# 3) Redis cache — production-grade
from langchain_community.cache import RedisCache
from redis import Redis
set_llm_cache(RedisCache(redis_=Redis.from_url('redis://localhost:6379')))

# Or async:
from langchain_community.cache import AsyncRedisCache
from redis.asyncio import Redis as AsyncRedis
set_llm_cache(AsyncRedisCache(redis_=AsyncRedis.from_url('redis://localhost:6379')))

# 4) Semantic cache — cache hits on similar prompts
from langchain_community.cache import RedisSemanticCache
from langchain_openai import OpenAIEmbeddings

set_llm_cache(RedisSemanticCache(
    redis_url='redis://localhost:6379',
    embedding=OpenAIEmbeddings(model='text-embedding-3-small'),
    score_threshold=0.2,                          # smaller = stricter match
))

# Now 'What is the capital of France?' and 'France's capital?' BOTH hit cache.
# Use carefully — too-loose threshold returns wrong answers.

# 5) Disabling cache for a single call
result = llm.invoke('What is today's news?', config={ 'cache': False })

# Per-LLM disable
uncached = ChatOpenAI(model='gpt-4o-mini', cache=False)

# 6) Cache namespacing — separate spaces per app / user
from langchain_community.cache import SQLiteCache
set_llm_cache(SQLiteCache(database_path='cache/app_a.db'))

# Or use Redis prefixes:
set_llm_cache(RedisCache(redis_=Redis.from_url('redis://localhost:6379/0'), ttl=3600))

# Many caches accept a TTL (Redis-based). In-memory + SQLite don't.

# 7) GPTCache — semantic + vector caches with rich features
# pip install gptcache
from langchain_community.cache import GPTCache
import gptcache
from gptcache.processor.pre import get_prompt
from gptcache.manager.factory import manager_factory
from gptcache.embedding import Onnx
from gptcache.similarity_evaluation.distance import SearchDistanceEvaluation

def init_gptcache(cache_obj, llm):
    cache_obj.init(
        pre_embedding_func=get_prompt,
        embedding_func=Onnx().to_embeddings,
        data_manager=manager_factory('sqlite,faiss', vector_params={'dimension': Onnx().dimension}),
        similarity_evaluation=SearchDistanceEvaluation(),
    )

set_llm_cache(GPTCache(init_gptcache))

# 8) Effective cost savings
# Repeated user queries (FAQs, common prompts) often hit > 50% cache rate.
# At GPT-4 prices that's significant; for embeddings it cuts latency too.

# 9) Cache embeddings too
from langchain.embeddings.cache import CacheBackedEmbeddings
from langchain.storage import LocalFileStore
from langchain_openai import OpenAIEmbeddings

store = LocalFileStore('./cache/')
base  = OpenAIEmbeddings(model='text-embedding-3-small')
cached_embeddings = CacheBackedEmbeddings.from_bytes_store(
    base, store, namespace=base.model,
)

# Use `cached_embeddings` in your vector store; identical texts skip the API.

# 10) Cache invalidation
from langchain_community.cache import InMemoryCache
import os

# Bust by prepending a version to your prompt template:
prompt = f'[v={os.getenv("PROMPT_VERSION", "1")}] Answer concisely. Q: {{q}}'
# Increment PROMPT_VERSION to invalidate.

# Or call .clear() / delete the SQLite file / FLUSHDB Redis.

# 11) Caching with streaming
llm = ChatOpenAI(streaming=True)
# Cached responses stream from cache (one chunk) instead of token-by-token.
# UX may differ; show a 'cached' badge or skip streaming UI for cached hits.

# 12) Multi-tenant caches
# Risk: User A's question stored, User B sees the cached answer when their question is semantically similar.
# For confidential apps:
#   • Disable cache for personalised prompts
#   • Namespace cache per tenant or user
#   • Hash the prompt + user_id as cache key in a custom cache class

class UserScopedCache:
    def __init__(self, backend): self.backend = backend
    def lookup(self, prompt, llm_string):
        return self.backend.lookup(f'{user_id}::{prompt}', llm_string)
    def update(self, prompt, llm_string, response):
        self.backend.update(f'{user_id}::{prompt}', llm_string, response)

# 13) Cache + LangSmith
# When tracing is on, cached responses STILL appear in LangSmith with metadata indicating cache hit.
# Easy to monitor cache hit rate over time.

# 14) Real-world checklist
# • Set cache once at app startup
# • Use Redis cache in prod (cross-replica + TTL)
# • SQLite for local dev (committable test snapshot)
# • Semantic cache for FAQ / docs Q&A; with strict threshold
# • Cache embeddings separately
# • Bust on prompt template changes via version prefix
# • Disable cache for personalised / time-sensitive prompts
# • Monitor hit rate, latency improvement, cost reduction

# 15) Common bugs
# • Semantic cache threshold too loose → wrong answers cached for unrelated prompts
# • Cache enabled but LLM has 'temperature' > 0 — first response cached even though it was stochastic; users get stale randomness
# • Cache key doesn't include model name → switching models returns OLD model's responses
# • In-memory cache in serverless functions — instance-local; no benefit
# • Cache with no TTL grows unbounded → use Redis TTL or periodic cleanup
# • PII in cached prompts → privacy risk; encrypt or namespace
# • Cached error responses → store success only; check status before update
# • Streaming UI + cache hit returns whole response at once → adjust UX expectations
# • set_llm_cache called inside a request handler → re-initialises every call; set ONCE at startup

from langchain.cache import InMemoryCache
from langchain.globals import set_llm_cache
set_llm_cache(InMemoryCache())  # same prompt → cached response