Crawl4AI 완벽 가이드 2026: GitHub 63k+ Stars 오픈소스 웹 크롤러로 LLM 데이터 파이프라인 구축하기

pip install crawl4ai
playwright install chromium

pip install crawl4ai[sync]

docker pull unclecode/crawl4ai:latest

import asyncio
from crawl4ai import AsyncWebCrawler

async def main():
    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(url="https://crawl4ai.com")
        print(result.markdown[:1000])

if __name__ == "__main__":
    asyncio.run(main())

crwl https://example.com -o markdown

from pydantic import BaseModel, Field

class ModelPricing(BaseModel):
    model_name: str = Field(..., description="모델명")
    input_cost: str = Field(..., description="입력 토큰당 1M 기준 비용")
    output_cost: str = Field(..., description="출력 토큰당 1M 기준 비용")

import os
import asyncio
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
from crawl4ai.extraction_strategy import LLMExtractionStrategy

async def main():
    browser_config = BrowserConfig(verbose=True)
    
    run_config = CrawlerRunConfig(
        word_count_threshold=1,
        extraction_strategy=LLMExtractionStrategy(
            provider="openai/gpt-4o",
            api_token=os.getenv('OPENAI_API_KEY'),
            schema=ModelPricing.model_json_schema(),
            extraction_type="schema",
            instruction=(
                "페이지 내 모든 모델명과 입력·출력 토큰 가격을 추출하라. "
                "형식: {'model_name': 'GPT-4o', 'input_cost': 'US$5.00 / 1M tokens', ...}"
            ),
            input_format="markdown",
            verbose=True
        ),
        cache_mode=CacheMode.BYPASS,
    )

    async with AsyncWebCrawler(config=browser_config) as crawler:
        result = await crawler.arun(
            url='https://openai.com/api/pricing/',
            config=run_config
        )
        print(result.extracted_content)

if __name__ == "__main__":
    asyncio.run(main())

import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy

async def main():
    config = CrawlerRunConfig(
        deep_crawl_strategy=BFSDeepCrawlStrategy(
            max_depth=2,
            include_external=False  # 외부 링크 제외
        ),
        scraping_strategy=LXMLWebScrapingStrategy(),
        verbose=True
    )

    async with AsyncWebCrawler() as crawler:
        results = await crawler.arun("https://docs.crawl4ai.com/", config=config)
        print(f"총 {len(results)}개 페이지 크롤링 완료")
        
        for r in results[:5]:
            print(f"URL: {r.url} | 깊이: {r.metadata.get('depth', 0)}")

if __name__ == "__main__":
    asyncio.run(main())

from crawl4ai.content_filter import BM25ContentFilter

filter = BM25ContentFilter(
    query="비동기 크롤러 설정 방법",
    threshold=0.1
)

docker run -p 8000:8000 \
  -e CRAWL4AI_API_TOKEN=your_jwt_secret \
  unclecode/crawl4ai:latest

curl -X POST http://localhost:8000/crawl \
  -H "Authorization: Bearer your_jwt_secret" \
  -d '{"url": "https://example.com", "output_format": "markdown"}'

browser_config = BrowserConfig(
    headless=True,
    proxy_config={
        "server": os.getenv("PROXY_SERVER"),
        "username": os.getenv("PROXY_USERNAME"),
        "password": os.getenv("PROXY_PASSWORD"),
    },
    verbose=True
)