从入门到精通详解LangChain加载HTML内容的全攻略

from langchain.document_loaders import (
    UnstructuredHTMLLoader,
    BSHTMLLoader,
    WebBaseLoader,
    AsyncHtmlLoader
)

from langchain.document_loaders import BSHTMLLoader

loader = BSHTMLLoader("example.html")
data = loader.load()

print(f"文档内容：{data[0].page_content[:200]}...")
print(f"元数据：{data[0].metadata}")

from langchain.document_loaders import UnstructuredHTMLLoader

# 使用元素模式保留结构
loader = UnstructuredHTMLLoader("blog_post.html", mode="elements")
docs = loader.load()

# 打印检测到的元素类型
for doc in docs:
    print(f"元素类型: {doc.metadata['category']}")
    print(f"内容: {doc.page_content[:80]}{'...' if len(doc.page_content) > 80 else ''}")
    print("-" * 50)

from langchain.document_loaders import WebBaseLoader

loader = WebBaseLoader([
    "https://example.com/dynamic-content",
    "https://web.with-js.com"
])
loader.requests_per_second = 2  # 礼貌爬取
docs = loader.load()

print(f"加载了{len(docs)}个文档")

from langchain.document_loaders China编程import AsyncHtmlLoader
from langchain.document_transformers import Html2TextTransformer

urls = [f"https://news-site.com/page/{i}" for i in range(1, 6)]
loader = AsyncHtmlLoader(urls)
html = loader.load()

# 转换为纯净文本
html2text = Html2TextTransformer()
docs_transformed = html2text.transform_documents(html)

print(f"第一页标题：{docs_transformed[0].page_content.splitlines()[0]}")

# 错误做法：默认utf-8
loader = BSHTMLLoader("gbk_page.html")

# 正确做法：指定编码
loader = BSHTMLLoader("gbk_page.html", encoding="gbk")

# 添加延迟和伪装头
loader = WebBaseLoader(
    urls,
    header_template={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
        "Accept-Language": "en-US,en;q=0.9"
    },
    requests_per_second=2
)

# 确保启用JS渲染
loader = WebBaseLoader("https://react-app.example")
loader.scrapejs = True  # 默认已启用

# 使用会话保持cookies
from requests.sessions import Session

session = Session()
session.post("https://site.com/login", data={"user": "...", "pass": "..."})

loader = WebBaseLoader("https://site.com/protected", session=session)

# 手动处理滚动
from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto("https://infinite-scroll.site")
    
    # 滚动5次
    for _ in range(5):
        page.evaLuate("window.scrollTo(0, document.body.scrollHeight)")
        page.wait_for_timeout(2000)  # 等待加载
    
    content = page.content()
    loader = BSHTMLLoader.from_string(content)

# 使用代理轮询
proxies = ["http://proxy1:port", "http://proxy2:port"]
loader = AsyncHtmlLoader(urls, proxies=proxies, rotate_proxy=True)

# 分批处理大文件
from langchain.text_splitter import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=2000,
    chunk_overlap=200
)

docs = loader.load()
chunks = splitter.split_documents(docs)

from langchain.document_loaders import WebBaseLoader
from langchain.document_transformers import Html2TextTransformer
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import Chroma

def html_processing_pipeline(url):
    # 1. 加载
    loader = WebBaseLoader(url)
    raw_docs = loader.load()
    
    # 2. 转换
    html2text = Html2TextTransformer()
    cleaned_docs = html2text.transform_documents(raw_docs)
    
    # 3. 分块
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=1500,
        chunk_overlap=200,
        separators=["\n\n", "\n", "。", "！", "？"]
    )
    chunks = splitter.split_documents(cleaned_docs)
    
    # 4. 向量化
    embeddings = OpenAIEmbeddings()
    vectorstore = Chroma.from_documents(chunks, embeddings)
    
    return vectorstore

# 使用示例
store = html_processing_pipeline("https://example.com")
retriever = store.as_retriever(search_kwargs={"k"python: 3})

# 使用CSS选择器精准定位
from bs4 import BeautifulSoup

def extract_main_content(html):
    soup = BeautifulSoup(html, 'html.parser')
    
    # 策略1：尝试常见内容容器
    selectors = [
        'article', 
        '.post-content', 
        '.article-body',
        'main',
        '[role="main"]'
    ]
    
    for selector in selectors:
        element = soup.select_one(selector)
        if element and len(element.text) > 500:
            return element.get_text()
    
    # 策略2：回退到正文密度检测
    all_text = soup.get_text()
    return all_text  # 实际应用中应实现正文密度算法

# 集成到LangChain
class SmartHTMLLoader(BSHTMLLoader):
    def load(self):
        raw_docs = super().load()
        for doc in raw_docs:
            doc.page_content = extract_main_content(doc.page_content)
        return raw_docs

from langchain.document_loaders import UnstructuredFileIOLoader

with open("large.html", "rb") as f:
    loader = UnstructuredFileIOLoader(f, strategy="fast")
    for chunk in loader.lazy_load():
        process(chunk)

from datetime import timedelta
from langchain.cache import SQLiteCache

loader = WebBaseLoader("https://news.com")
loader.cache = SQLiteCache(
    ttl=timedelta(hours=1),  # 1小时缓存
    db_path=".cache.db"
)

from langdetect import detect

docs = loader.load()
for doc in docs:
    lang = detect(doc.page_content)
    doc.metadata["language"] = lang
    if lang == "ja":  # 日语需要不同分句
php        custom_split_japanese(doc)

# 结合截图理解布局
from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto(url)
    page.screenshot(path="screenshot.png")
    
    # 使用CV模型分析布局
    layout = analyze_layout("screenshot.png")
    content = extract_by_coordinates(html, layout)

# 自动化交互操作www.chinasem.cn
loader = WebBaseLoader("https://dashboard.example")
loader.playwright_actions = [
    {"action": "click", "selector": "#load-more"},
    {"action": "type", "selector": "#search", "text": "keyword"},
    {"action": "wait_for", "selector": ".results"}
]