Colly: 25,302 GitHub Stars — 벤치마크 Go 웹 스크래핑 프레임워크 2026

# 프로젝트 초기화
mkdir colly-scraper && cd colly-scraper
go mod init github.com/youruser/colly-scraper

# Colly v2 설치
go get github.com/gocolly/colly/v2

# 설치 확인
go list -m github.com/gocolly/colly/v2

package main

import (
	"fmt"
	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector()

	// 모든 링크 제목 추출
	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Attr("href")
		text := e.Text
		fmt.Printf("링크: %s | 텍스트: %s\n", link, text)
	})

	c.OnRequest(func(r *colly.Request) {
		fmt.Println("방문 중:", r.URL.String())
	})

	c.OnError(func(r *colly.Response, err error) {
		fmt.Printf("오류 %d: %v\n", r.StatusCode, err)
	})

	c.Visit("https://go-colly.org/")
}

go run main.go

FROM golang:1.24-alpine AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -o scraper main.go

FROM alpine:latest
RUN apk --no-cache add ca-certificates
WORKDIR /root/
COPY --from=builder /app/scraper .
CMD ["./scraper"]

# 빌드 및 실행
docker build -t colly-scraper .
docker run --rm colly-scraper

version: '3.8'
services:
  scraper:
    build: .
    depends_on:
      - redis
    environment:
      - REDIS_URL=redis:6379
  redis:
    image: redis:7-alpine
    volumes:
      - redis-data:/data
  volumes:
    redis-data:

package main

import (
	"github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/storage"
)

func main() {
	c := colly.NewCollector()

	// Redis를 영구 스토리지로 사용
	redisStore := &storage.RedisStorage{
		Address:  "redis:6379",
		Password: "",
		DB:       0,
		Prefix:   "colly",
	}

	if err := redisStore.Open(); err != nil {
		panic(err)
	}
	defer redisStore.Close()

	c.SetStorage(redisStore)

	c.OnHTML("h1", func(e *colly.HTMLElement) {
		fmt.Println("제목:", e.Text)
	})

	c.Visit("https://example.com")
}

package main

import (
	"github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/proxy"
)

func main() {
	c := colly.NewCollector()

	// 로테이션 프록시 설정
	rp, err := proxy.RoundRobinProxySwitcher(
		"http://user:pass@proxy1.webshare.io:80",
		"http://user:pass@proxy2.webshare.io:80",
		"http://user:pass@proxy3.webshare.io:80",
	)
	if err != nil {
		panic(err)
	}
	c.SetProxyFunc(rp)

	// 대상 서버 존중
	c.Limit(&colly.LimitRule{
		DomainGlob:  "*",
		Parallelism: 10,
		Delay:       1 * time.Second,
	})

	c.Visit("https://example.com")
}

package main

import (
	"github.com/PuerkitoBio/goquery"
	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector()

	c.OnHTML("article", func(e *colly.HTMLElement) {
		// 기본 goquery 선택 접근
		dom := e.DOM

		// 복잡한 탐색
		title := dom.Find("h2").First().Text()
		author := dom.Find(".author").Text()

		// 형제 노드 탐색
		dom.Find("p").Siblings().Each(func(i int, s *goquery.Selection) {
			fmt.Printf("형제 %d: %s\n", i, s.Text())
		})

		// 부모 노드 조회
		category := dom.Parent().Find(".category").Text()
		fmt.Printf("기사: %s 작성자: %s [%s]\n", title, author, category)
	})

	c.Visit("https://news.ycombinator.com")
}

package main

import (
	"context"
	"fmt"
	"time"

	"github.com/chromedp/chromedp"
	"github.com/gocolly/colly/v2"
)

func renderWithChrome(url string) string {
	ctx, cancel := chromedp.NewContext(context.Background())
	defer cancel()

	ctx, cancel = context.WithTimeout(ctx, 15*time.Second)
	defer cancel()

	var html string
	err := chromedp.Run(ctx,
		chromedp.Navigate(url),
		chromedp.WaitReady("body"),
		chromedp.OuterHTML("html", &html),
	)
	if err != nil {
		return ""
	}
	return html
}

func main() {
	// 먼저 Chrome으로 페이지 렌더링, 그 다음 Colly로 파싱
	htmlContent := renderWithChrome("https://spa-example.com")

	c := colly.NewCollector()
	// 렌더링된 HTML 파싱...
	fmt.Println("렌더링 길이:", len(htmlContent))
}

package main

import (
	"time"
	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector(
		colly.AllowedDomains("example.com"),
		colly.UserAgent("MyBot/1.0 (+https://mysite.com/bot)"),
	)

	// 엄격한 도메인별 속도 제한
	c.Limit(&colly.LimitRule{
		DomainGlob:  "*example.com",
		Parallelism: 5,
		Delay:       2 * time.Second,
		RandomDelay: 500 * time.Millisecond,
	})

	// robots.txt 준수
	c.AllowURLRevisit = false

	c.Visit("https://example.com/products")
}

package main

import (
	"github.com/gocolly/colly/v2"
	"github.com/gocolly/colly/v2/queue"
)

func main() {
	c := colly.NewCollector()

	// Redis 기반 큐 생성
	q, _ := queue.New(100, &queue.RedisStorage{
		Address: "redis:6379",
		DB:      0,
	})

	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
		link := e.Request.AbsoluteURL(e.Attr("href"))
		if link != "" {
			q.AddURL(link)
		}
	})

	c.OnHTML("article", func(e *colly.HTMLElement) {
		title := e.ChildText("h1")
		body := e.ChildText("p")
		saveToDatabase(title, body)
	})

	q.AddURL("https://example.com/start")
	q.Run(c)
}

package main

import (
	"net/http"
	"time"
	"github.com/gocolly/colly/v2"
)

func main() {
	c := colly.NewCollector()

	// 기본 HTTP 클라이언트 교체
	c.WithTransport(&http.Transport{
		MaxIdleConns:        100,
		MaxIdleConnsPerHost: 10,
		IdleConnTimeout:     30 * time.Second,
		DisableCompression:  false,
	})

	c.SetRequestTimeout(15 * time.Second)

	// 실패한 요청 재시도
	c.OnError(func(r *colly.Response, err error) {
		if r.StatusCode >= 500 {
			// 서버 오류 5초 후 한 번 재시도
			time.Sleep(5 * time.Second)
			r.Request.Retry()
		}
	})

	c.Visit("https://example.com")
}

package main

import (
	"encoding/json"
	"fmt"
	"github.com/gocolly/colly/v2"
)

type Product struct {
	Name  string `selector:"h1.product-title"`
	Price string `selector:"span.price"`
	SKU   string `selector:"meta[itemprop=sku]" attr:"content"`
}

func main() {
	c := colly.NewCollector()

	c.OnHTML("div.product", func(e *colly.HTMLElement) {
		var p Product
		e.Unmarshal(&p)
		data, _ := json.MarshalIndent(p, "", "  ")
		fmt.Println(string(data))
	})

	c.Visit("https://shop.example.com/item/123")
}