SigNoz：以Datadog 10%成本替代的开源APM —— 分布式追踪设置指南2026

应用（OTel SDK） → OTLP/gRPC → SigNoz Otel Collector
                                        ↓
                              ┌──────────────────┐
                              │      Kafka       │
                              └──────┬─────┬─────┘
                                     ↓     ↓
                               ClickHouse  Druid
                               （追踪/    （指标）
                                日志）
                                     ↓     ↓
                              查询服务（Go）
                                     ↓
                                React前端

# 不需要供应商特定的SDK —— 仅标准OTel
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

# 配置指向SigNoz收集器的OTLP导出器
otlp_exporter = OTLPSpanExporter(
    endpoint="http://signoz-otel-collector:4317",
    insecure=True
)

provider = TracerProvider()
processor = BatchSpanProcessor(otlp_exporter)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

# 1. 克隆SigNoz仓库
git clone -b main https://github.com/SigNoz/signoz.git
cd signoz/deploy/docker

# 2. 运行安装脚本
./install.sh

# 脚本将：
# - 检查Docker和Docker Compose版本
# - 拉取所有所需镜像（ClickHouse、Kafka、查询服务、前端）
# - 启动所有服务
# - 打印访问URL

# 1. 添加SigNoz Helm仓库
helm repo add signoz https://charts.signoz.io
helm repo update

# 2. 安装SigNoz到集群
kubectl create namespace signoz
helm install signoz signoz/signoz \
  --namespace signoz \
  --set otelCollector.endpoint.host=signoz-otel-collector.signoz.svc.cluster.local \
  --set clickhouse.persistence.size=100Gi

# 3. 等待所有Pod就绪
kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=signoz -n signoz --timeout=300s

# 4. 端口转发前端
kubectl port-forward svc/signoz-frontend 3301:3301 -n signoz

# docker-compose.production.yml
version: "3.8"
services:
  signoz-frontend:
    image: signoz/frontend:0.76.0
    restart: unless-stopped
    ports:
      - "3301:3301"
    depends_on:
      - signoz-query-service

  signoz-query-service:
    image: signoz/query-service:0.76.0
    restart: unless-stopped
    environment:
      - ClickHouseUrl=tcp://clickhouse:9000
      - DruidUrl=http://druid-router:8888
      - STORAGE=clickhouse
    depends_on:
      - clickhouse
      - druid

  signoz-otel-collector:
    image: signoz/signoz-otel-collector:0.76.0
    restart: unless-stopped
    ports:
      - "4317:4317"    # OTLP gRPC
      - "4318:4318"    # OTLP HTTP
      - "8889:8889"    # Prometheus指标
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    command: ["--config", "/etc/otel-collector-config.yaml"]

  clickhouse:
    image: clickhouse/clickhouse-server:24.3-alpine
    restart: unless-stopped
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    volumes:
      - clickhouse-data:/var/lib/clickhouse
    environment:
      - CLICKHOUSE_DB=signoz_metrics
      - CLICKHOUSE_USER=admin
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}

  zookeeper:
    image: zookeeper:3.9
    restart: unless-stopped
    volumes:
      - zookeeper-data:/data
      - zookeeper-logs:/datalog

  kafka:
    image: bitnami/kafka:3.7
    restart: unless-stopped
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
    volumes:
      - kafka-data:/bitnami/kafka
    depends_on:
      - zookeeper

volumes:
  clickhouse-data:
  kafka-data:
  zookeeper-data:
  zookeeper-logs:

# 检查所有容器是否运行中
docker ps --format "table {{.Names}}\t{{.Status}}"

# 预期输出：
# NAMES                        STATUS
# docker-clickhouse-1          Up 2 minutes (healthy)
# docker-kafka-1               Up 2 minutes
# docker-signoz-frontend-1     Up 2 minutes
# docker-signoz-query-service-1 Up 2 minutes
# docker-zookeeper-1           Up 2 minutes
# docker-signoz-otel-collector-1 Up 2 minutes

# 测试健康端点
curl http://localhost:3301/api/v1/health
# 输出：{"status":"ok"}

# Node.js —— 零代码更改
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" \
OTEL_RESOURCE_ATTRIBUTES="service.name=payment-service" \
npx @opentelemetry/auto-instrumentations-node ./server.js

# Python —— 零代码更改
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" \
OTEL_RESOURCE_ATTRIBUTES="service.name=order-service" \
opentelemetry-instrument python manage.py runserver

# Java —— 添加agent JVM参数
java -javaagent:opentelemetry-javaagent.jar \
  -Dotel.exporter.otlp.endpoint=http://localhost:4317 \
  -Dotel.resource.attributes=service.name=inventory-service \
  -jar application.jar

# Go —— 添加自动埋点包
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" \
OTEL_RESOURCE_ATTRIBUTES="service.name=api-gateway" \
go run main.go

# 使用手动埋点的Python Flask
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.resources import Resource, SERVICE_NAME
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from flask import Flask

# 配置tracer
resource = Resource.create({SERVICE_NAME: "payment-service"})
provider = TracerProvider(resource=resource)
otlp_exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True)
processor = BatchSpanProcessor(otlp_exporter)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

app = Flask(__name__)
tracer = trace.get_tracer(__name__)

@app.route("/process-payment", methods=["POST"])
def process_payment():
    with tracer.start_as_current_span("process_payment") as span:
        span.set_attribute("payment.amount", 149.00)
        span.set_attribute("payment.currency", "USD")

        with tracer.start_as_current_span("validate_card"):
            # 卡验证逻辑
            pass

        with tracer.start_as_current_span("charge_stripe"):
            # Stripe API调用
            pass

        return {"status": "success"}

# 通过API创建自定义仪表板
curl -X POST http://localhost:3301/api/v1/dashboards \
  -H "Content-Type: application/json" \
  -d '{
    "title": "支付服务健康度",
    "description": "支付处理的关键指标",
    "panels": [
      {
        "title": "请求速率",
        "query": "SELECT count() FROM signoz_traces.signoz_index_v2 WHERE serviceName = '\''payment-service'\''",
        "widgetType": "graph",
        "timeRange": "1h"
      },
      {
        "title": "P95延迟",
        "query": "SELECT histogramQuantile(0.95)(durationNano) / 1000000 FROM signoz_traces.signoz_index_v2 WHERE serviceName = '\''payment-service'\''",
        "widgetType": "value",
        "unit": "ms"
      },
      {
        "title": "错误率 %",
        "query": "SELECT (countIf(statusCode = 2) * 100.0 / count()) FROM signoz_traces.signoz_index_v2 WHERE serviceName = '\''payment-service'\''",
        "widgetType": "gauge"
      }
    ]
  }'

# docker-compose.ha.yml —— 带ZooKeeper的多节点ClickHouse
services:
  clickhouse-1:
    image: clickhouse/clickhouse-server:24.3-alpine
    volumes:
      - clickhouse1-data:/var/lib/clickhouse
      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/cluster.xml
    environment:
      - CLICKHOUSE_USER=admin
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}

  clickhouse-2:
    image: clickhouse/clickhouse-server:24.3-alpine
    volumes:
      - clickhouse2-data:/var/lib/clickhouse
      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/cluster.xml
    environment:
      - CLICKHOUSE_USER=admin
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}

  clickhouse-3:
    image: clickhouse/clickhouse-server:24.3-alpine
    volumes:
      - clickhouse3-data:/var/lib/clickhouse
      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/cluster.xml
    environment:
      - CLICKHOUSE_USER=admin
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}

  # ClickHouse的Nginx负载均衡器
  clickhouse-lb:
    image: nginx:alpine
    volumes:
      - ./nginx-clickhouse.conf:/etc/nginx/nginx.conf
    ports:
      - "8123:8123"
      - "9000:9000"

# ClickHouse S3备份配置
<clickhouse>
  <storage_configuration>
    <disks>
      <s3_disk>
        <type>s3</type>
        <endpoint>https://s3.amazonaws.com/my-bucket/signoz/</endpoint>
        <access_key_id>${AWS_ACCESS_KEY}</access_key_id>
        <secret_access_key>${AWS_SECRET_KEY}</secret_access_key>
      </s3_disk>
    </disks>
    <policies>
      <tiered_storage>
        <volumes>
          <hot>
            <disk>default</disk>
            <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
          </hot>
          <cold>
            <disk>s3_disk</disk>
          </cold>
        </volumes>
      </tiered_storage>
    </policies>
  </storage_configuration>
</clickhouse>

# alert-rules.yml —— SigNoz告警管理器规则
groups:
  - name: payment_service_alerts
    rules:
      - alert: HighErrorRate
        expr: |
          (
            sum(rate(signoz_calls_total{service_name="payment-service",status_code="STATUS_CODE_ERROR"}[5m]))
            /
            sum(rate(signoz_calls_total{service_name="payment-service"}[5m]))
          ) > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "支付服务错误率 > 5%"
          description: "错误率为 {{ $value }}"

      - alert: HighP95Latency
        expr: histogramQuantile(0.95)(rate(signoz_latency_bucket{service_name="payment-service"}[5m])) > 500000000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "支付服务P95延迟 > 500ms"
          description: "P95延迟为 {{ $value }}ns"

      - alert: LogErrorSpike
        expr: rate(signoz_logs_total{severity="ERROR"}[5m]) > 100
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "检测到日志错误峰值"
          description: "{{ $value }} 错误/分钟"

# signoz-otel-collector-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: signoz-otel-collector
  namespace: signoz
spec:
  ports:
    - name: otlp-grpc
      port: 4317
      protocol: TCP
    - name: otlp-http
      port: 4318
      protocol: TCP
  selector:
    app.kubernetes.io/name: otel-collector

---
# 通过添加OTel环境变量为Deployment埋点
apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-service
spec:
  template:
    spec:
      containers:
        - name: payment-service
          image: payment-service:1.2.3
          env:
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: "http://signoz-otel-collector.signoz.svc.cluster.local:4317"
            - name: OTEL_RESOURCE_ATTRIBUTES
              value: "service.name=payment-service,service.namespace=production"
            - name: OTEL_TRACES_SAMPLER
              value: "parentbased_traceidratio"
            - name: OTEL_TRACES_SAMPLER_ARG
              value: "0.1"  # 采样10%的追踪

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 1000
    policies:
      - name: errors
        type: status_code
        status_code: {status_codes: [ERROR]}
      - name: slow_requests
        type: latency
        latency: {threshold_ms: 500}
      - name: probabilistic
        type: probabilistic
        probabilistic: {sampling_percentage: 10}

exporters:
  clickhousetraces:
    datasource: tcp://clickhouse:9000
    database: signoz_traces

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [tail_sampling]
      exporters: [clickhousetraces]

receivers:
  prometheus:
    config:
      scrape_configs:
        - job_name: 'my-app'
          static_configs:
            - targets: ['my-app:9090']