Colly: 25,302 GitHub Stars — Benchmark Go 网页抓取框架 2026

# 初始化项目
mkdir colly-scraper && cd colly-scraper
go mod init github.com/youruser/colly-scraper

# 安装 Colly v2
go get github.com/gocolly/colly/v2

# 验证安装
go list -m github.com/gocolly/colly/v2

package main

import (
	"fmt"
	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector()

	// 提取所有链接标题
	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Attr("href")
		text := e.Text
		fmt.Printf("链接: %s | 文本: %s\n", link, text)
	})

	c.OnRequest(func(r *colly.Request) {
		fmt.Println("正在访问:", r.URL.String())
	})

	c.OnError(func(r *colly.Response, err error) {
		fmt.Printf("错误 %d: %v\n", r.StatusCode, err)
	})

	c.Visit("https://go-colly.org/")
}

go run main.go

FROM golang:1.24-alpine AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o scraper main.go

FROM alpine:latest
RUN apk --no-cache add ca-certificates
WORKDIR /root/
COPY --from=builder /app/scraper .
CMD ["./scraper"]

# 构建并运行
docker build -t colly-scraper .
docker run --rm colly-scraper

version: '3.8'
services:
  scraper:
    build: .
    depends_on:
      - redis
    environment:
      - REDIS_URL=redis:6379
  redis:
    image: redis:7-alpine
    volumes:
      - redis-data:/data
  volumes:
    redis-data:

package main

import (
	"github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/storage"
)

func main() {
	c := colly.NewCollector()

	// 使用 Redis 作为持久化存储
	redisStore := &storage.RedisStorage{
		Address:  "redis:6379",
		Password: "",
		DB:       0,
		Prefix:   "colly",
	}

	if err := redisStore.Open(); err != nil {
		panic(err)
	}
	defer redisStore.Close()

	c.SetStorage(redisStore)

	c.OnHTML("h1", func(e *colly.HTMLElement) {
		fmt.Println("标题:", e.Text)
	})

	c.Visit("https://example.com")
}

package main

import (
	"github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/proxy"
)

func main() {
	c := colly.NewCollector()

	// 设置轮换代理
	rp, err := proxy.RoundRobinProxySwitcher(
		"http://user:pass@proxy1.webshare.io:80",
		"http://user:pass@proxy2.webshare.io:80",
		"http://user:pass@proxy3.webshare.io:80",
	)
	if err != nil {
		panic(err)
	}
	c.SetProxyFunc(rp)

	// 尊重目标服务器
	c.Limit(&colly.LimitRule{
		DomainGlob:  "*",
		Parallelism: 10,
		Delay:       1 * time.Second,
	})

	c.Visit("https://example.com")
}

package main

import (
	"github.com/PuerkitoBio/goquery"
	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector()

	c.OnHTML("article", func(e *colly.HTMLElement) {
		// 访问底层 goquery 选择
		dom := e.DOM

		// 复杂遍历
		title := dom.Find("h2").First().Text()
		author := dom.Find(".author").Text()

		// 兄弟节点遍历
		dom.Find("p").Siblings().Each(func(i int, s *goquery.Selection) {
			fmt.Printf("兄弟节点 %d: %s\n", i, s.Text())
		})

		// 父节点查找
		category := dom.Parent().Find(".category").Text()
		fmt.Printf("文章: %s 作者: %s [%s]\n", title, author, category)
	})

	c.Visit("https://news.ycombinator.com")
}

package main

import (
	"context"
	"fmt"
	"time"

	"github.com/chromedp/chromedp"
	"github.com/gocolly/colly/v2"
)

func renderWithChrome(url string) string {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	ctx, cancel = context.WithTimeout(ctx, 15*time.Second)
	defer cancel()

	var html string
	err := chromedp.Run(ctx,
		chromedp.Navigate(url),
		chromedp.WaitReady("body"),
		chromedp.OuterHTML("html", &html),
	)
	if err != nil {
		return ""
	}
	return html
}

func main() {
	// 先用 Chrome 渲染页面，再用 Colly 解析
	htmlContent := renderWithChrome("https://spa-example.com")

	c := colly.NewCollector()
	// 解析渲染后的 HTML...
	fmt.Println("渲染后长度:", len(htmlContent))
}

package main

import (
	"time"
	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector(
		colly.AllowedDomains("example.com"),
		colly.UserAgent("MyBot/1.0 (+https://mysite.com/bot)"),
	)

	// 严格的每域名速率限制
	c.Limit(&colly.LimitRule{
		DomainGlob:  "*example.com",
		Parallelism: 5,
		Delay:       2 * time.Second,
		RandomDelay: 500 * time.Millisecond,
	})

	// 遵守 robots.txt
	c.AllowURLRevisit = false

	c.Visit("https://example.com/products")
}

package main

import (
	"github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/queue"
)

func main() {
	c := colly.NewCollector()

	// 创建 Redis 支持的队列
	q, _ := queue.New(100, &queue.RedisStorage{
		Address: "redis:6379",
		DB:      0,
	})

	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Request.AbsoluteURL(e.Attr("href"))
		if link != "" {
			q.AddURL(link)
		}
	})

	c.OnHTML("article", func(e *colly.HTMLElement) {
		title := e.ChildText("h1")
		body := e.ChildText("p")
		saveToDatabase(title, body)
	})

	q.AddURL("https://example.com/start")
	q.Run(c)
}

package main

import (
	"net/http"
	"time"
	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector()

	// 替换默认 HTTP 客户端
	c.WithTransport(&http.Transport{
		MaxIdleConns:        100,
		MaxIdleConnsPerHost: 10,
		IdleConnTimeout:     30 * time.Second,
		DisableCompression:  false,
	})

	c.SetRequestTimeout(15 * time.Second)

	// 重试失败请求
	c.OnError(func(r *colly.Response, err error) {
		if r.StatusCode >= 500 {
			// 服务器错误 5 秒后重试一次
			time.Sleep(5 * time.Second)
			r.Request.Retry()
		}
	})

	c.Visit("https://example.com")
}

package main

import (
	"encoding/json"
	"fmt"
	"github.com/gocolly/colly/v2"
)

type Product struct {
	Name  string `selector:"h1.product-title"`
	Price string `selector:"span.price"`
	SKU   string `selector:"meta[itemprop=sku]" attr:"content"`
}

func main() {
	c := colly.NewCollector()

	c.OnHTML("div.product", func(e *colly.HTMLElement) {
		var p Product
		e.Unmarshal(&p)
		data, _ := json.MarshalIndent(p, "", "  ")
		fmt.Println(string(data))
	})

	c.Visit("https://shop.example.com/item/123")
}