Crawl4AI 深度实战教程：2026 年 GitHub 最火开源爬虫，零成本搭建 LLM 数据采集与 RAG 知识库

pip install crawl4ai
playwright install chromium

pip install crawl4ai[sync]

docker pull unclecode/crawl4ai:latest

import asyncio
from crawl4ai import AsyncWebCrawler

async def main():
    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(url="https://crawl4ai.com")
        print(result.markdown[:1000])

if __name__ == "__main__":
    asyncio.run(main())

# Crawl4AI: Open-Source LLM-Friendly Web Crawler

Crawl4AI turns the web into clean, LLM ready Markdown for RAG, agents, and data pipelines...

crwl https://example.com -o markdown

from pydantic import BaseModel, Field

class OpenAIModelFee(BaseModel):
    model_name: str = Field(..., description="模型名称")
    input_fee: str = Field(..., description="输入 Token 单价")
    output_fee: str = Field(..., description="输出 Token 单价")

import os
import asyncio
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
from crawl4ai.extraction_strategy import LLMExtractionStrategy

async def main():
    browser_config = BrowserConfig(verbose=True)
    
    run_config = CrawlerRunConfig(
        word_count_threshold=1,
        extraction_strategy=LLMExtractionStrategy(
            provider="openai/gpt-4o",
            api_token=os.getenv('OPENAI_API_KEY'),
            schema=OpenAIModelFee.model_json_schema(),
            extraction_type="schema",
            instruction=(
                "从页面内容提取所有模型名称及其输入/输出 Token 费用。"
                "格式示例：{'model_name': 'GPT-4o', 'input_fee': 'US$5.00 / 1M tokens', ...}"
            ),
            input_format="markdown",
            verbose=True
        ),
        cache_mode=CacheMode.BYPASS,
    )

    async with AsyncWebCrawler(config=browser_config) as crawler:
        result = await crawler.arun(
            url='https://openai.com/api/pricing/',
            config=run_config
        )
        print(result.extracted_content)

if __name__ == "__main__":
    asyncio.run(main())

extraction_strategy = LLMExtractionStrategy(
    provider="groq/deepseek-r1-distill-llama-70b",
    api_token=os.getenv('GROQ_API_KEY'),
    schema=ProductSchema.model_json_schema(),
    extraction_type="schema",
    instruction="提取产品名称、图片 URL、描述、评分和评论数。",
    input_format="markdown",
    verbose=True
)

import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy

async def main():
    config = CrawlerRunConfig(
        deep_crawl_strategy=BFSDeepCrawlStrategy(
            max_depth=2,
            include_external=False  # 不爬出站链接
        ),
        scraping_strategy=LXMLWebScrapingStrategy(),
        verbose=True
    )

    async with AsyncWebCrawler() as crawler:
        results = await crawler.arun("https://docs.crawl4ai.com/", config=config)
        print(f"共抓取 {len(results)} 个页面")
        
        for r in results[:3]:
            print(f"URL: {r.url} | 深度: {r.metadata.get('depth', 0)}")

if __name__ == "__main__":
    asyncio.run(main())

from crawl4ai.content_filter import BM25ContentFilter

filter = BM25ContentFilter(
    query="异步爬虫配置方法",
    threshold=0.1  # 相似度阈值
)

docker run -p 8000:8000 \
  -e CRAWL4AI_API_TOKEN=your_secret \
  unclecode/crawl4ai:latest

curl -X POST http://localhost:8000/crawl \
  -H "Authorization: Bearer your_secret" \
  -d '{"url": "https://example.com", "output_format": "markdown"}'

browser_config = BrowserConfig(
    headless=True,
    proxy_config={
        "server": os.getenv("PROXY_SERVER"),
        "username": os.getenv("PROXY_USERNAME"),
        "password": os.getenv("PROXY_PASSWORD"),
    },
    verbose=True
)