Document Loaders

from langchain_community.document_loaders import (
    TextLoader, DirectoryLoader, PyPDFLoader, UnstructuredFileLoader,
    WebBaseLoader, RecursiveUrlLoader,
    GitLoader, NotionDirectoryLoader, ConfluenceLoader,
    CSVLoader, JSONLoader, BSHTMLLoader,
)
from langchain_text_splitters import RecursiveCharacterTextSplitter, MarkdownHeaderTextSplitter

# 1) Plain text file
loader = TextLoader('./docs/spec.txt', encoding='utf-8')
docs = loader.load()
print(docs[0].page_content[:200])
print(docs[0].metadata)        # { 'source': './docs/spec.txt' }

# 2) Directory — recursive, filtered
loader = DirectoryLoader(
    './docs',
    glob='**/*.md',
    loader_cls=TextLoader,
    show_progress=True,
    use_multithreading=True,
)
docs = loader.load()

# 3) PDFs
loader = PyPDFLoader('paper.pdf')
docs = loader.load()
# One Document per page; metadata has 'page' number

# For ranked layout + images + tables, try Unstructured:
loader = UnstructuredFileLoader('paper.pdf', mode='elements')
elements = loader.load()
# Elements have categories: 'NarrativeText', 'Title', 'ListItem', 'Table', ...

# 4) Web pages
loader = WebBaseLoader([
    'https://example.com/docs/intro',
    'https://example.com/docs/usage',
])
docs = loader.load()

# Crawl a whole site
loader = RecursiveUrlLoader(
    url='https://example.com/docs/',
    max_depth=2,
    prevent_outside=True,
    use_async=True,
)
docs = loader.load()

# 5) Code (Git repos)
loader = GitLoader(
    clone_url='https://github.com/me/myapp.git',
    repo_path='./tmp/myapp',
    branch='main',
    file_filter=lambda f: f.endswith(('.py', '.md', '.ts', '.go')),
)
docs = loader.load()

# 6) Notion / Confluence / Google Drive — when team docs live there
loader = NotionDirectoryLoader('./exports/notion-export')
loader = ConfluenceLoader(url='https://corp.atlassian.net/wiki', username='x', api_key='y', space_key='ENG')

# 7) CSV — one Document per row by default
loader = CSVLoader(
    './data/faqs.csv',
    source_column='question',
    metadata_columns=['category', 'tags'],
)
docs = loader.load()

# 8) JSON / JSONL via jq path
loader = JSONLoader(
    file_path='./data/events.jsonl',
    jq_schema='.message',
    text_content=False,
    metadata_func=lambda r, m: {**m, 'user_id': r.get('user_id'), 'ts': r.get('ts')},
)

# 9) HTML files
loader = BSHTMLLoader('./docs/page.html')
docs = loader.load()
# Or strip to main content + use Unstructured for ranked layout

# 10) Custom metadata — make retrieval smarter
for d in docs:
    d.metadata.update({
        'source':   d.metadata.get('source', 'unknown'),
        'category': 'support',
        'updated':  datetime.utcnow().isoformat(),
    })

# 11) Splitting — chunk before embedding
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separators=['\n## ', '\n### ', '\n\n', '\n', '. ', ' '],
    length_function=len,
    add_start_index=True,
)
chunks = splitter.split_documents(docs)

# For Markdown / docs with clear hierarchy — split on headings + carry metadata:
md_splitter = MarkdownHeaderTextSplitter(headers_to_split_on=[
    ('#', 'h1'), ('##', 'h2'), ('###', 'h3'),
])
header_splits = md_splitter.split_text(text)

# 12) Best practices
# - Set chunk_size by tokens not characters when you'll feed an LLM (use tiktoken).
# - Smaller chunks = more precise retrieval; bigger chunks = more context per chunk.
# - chunk_overlap of ~10-20% prevents losing concepts at boundaries.
# - Keep rich metadata (source URL, title, last updated) — show citations in your UI.
# - For binary / scanned PDFs: OCR first (pytesseract, AWS Textract).
# - For huge corpora: paginate; embed in batches; checkpoint progress.

# 13) Loader patterns by source type
#   Markdown docs / README     — TextLoader + DirectoryLoader + MarkdownHeaderTextSplitter
#   PDFs                        — PyPDFLoader (fast) or Unstructured (layout-aware)
#   Wiki / KB                   — Confluence / Notion / Slack loaders
#   Code base                   — GitLoader + RecursiveCharacterTextSplitter with code separators
#   Web docs                    — WebBaseLoader / RecursiveUrlLoader + bs4 parsing
#   Structured data             — CSVLoader / JSONLoader
#   Multimodal                  — Unstructured ingestion + a multimodal embedding model

from langchain_community.document_loaders import PyPDFLoader, WebBaseLoader
docs1 = PyPDFLoader('handbook.pdf').load()
docs2 = WebBaseLoader('https://example.com/blog').load()