SigNoz: APM mã nguồn mở thay thế Datadog với 10% chi phí — Hướng dẫn thiết lập Distributed Tracing 2026

Ứng dụng (OTel SDK) → OTLP/gRPC → SigNoz Otel Collector
                                        ↓
                              ┌──────────────────┐
                              │      Kafka       │
                              └──────┬─────┬─────┘
                                     ↓     ↓
                               ClickHouse  Druid
                               (traces/    (metrics)
                                logs)
                                     ↓     ↓
                              Query Service (Go)
                                     ↓
                                React Frontend

# Không cần SDK dành riêng cho vendor — chỉ cần OTel chuẩn
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

# Cấu hình OTLP exporter trỏ đến SigNoz collector
otlp_exporter = OTLPSpanExporter(
    endpoint="http://signoz-otel-collector:4317",
    insecure=True
)

provider = TracerProvider()
processor = BatchSpanProcessor(otlp_exporter)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

# 1. Clone repository SigNoz
git clone -b main https://github.com/SigNoz/signoz.git
cd signoz/deploy/docker

# 2. Chạy script cài đặt
./install.sh

# Script sẽ:
# - Kiểm tra Docker và Docker Compose versions
# - Pull tất cả các images cần thiết (ClickHouse, Kafka, Query Service, Frontend)
# - Khởi động tất cả services
# - In URL truy cập

# 1. Thêm Helm repository SigNoz
helm repo add signoz https://charts.signoz.io
helm repo update

# 2. Cài đặt SigNoz vào cluster
kubectl create namespace signoz
helm install signoz signoz/signoz \
  --namespace signoz \
  --set otelCollector.endpoint.host=signoz-otel-collector.signoz.svc.cluster.local \
  --set clickhouse.persistence.size=100Gi

# 3. Đợi tất cả pods sẵn sàng
kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=signoz -n signoz --timeout=300s

# 4. Port-forward frontend
kubectl port-forward svc/signoz-frontend 3301:3301 -n signoz

# docker-compose.production.yml
version: "3.8"
services:
  signoz-frontend:
    image: signoz/frontend:0.76.0
    restart: unless-stopped
    ports:
      - "3301:3301"
    depends_on:
      - signoz-query-service

  signoz-query-service:
    image: signoz/query-service:0.76.0
    restart: unless-stopped
    environment:
      - ClickHouseUrl=tcp://clickhouse:9000
      - DruidUrl=http://druid-router:8888
      - STORAGE=clickhouse
    depends_on:
      - clickhouse
      - druid

  signoz-otel-collector:
    image: signoz/signoz-otel-collector:0.76.0
    restart: unless-stopped
    ports:
      - "4317:4317"    # OTLP gRPC
      - "4318:4318"    # OTLP HTTP
      - "8889:8889"    # Prometheus metrics
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    command: ["--config", "/etc/otel-collector-config.yaml"]

  clickhouse:
    image: clickhouse/clickhouse-server:24.3-alpine
    restart: unless-stopped
    ulimits:
      nofile:
        soft: 262144
        hard: 262144
    volumes:
      - clickhouse-data:/var/lib/clickhouse
    environment:
      - CLICKHOUSE_DB=signoz_metrics
      - CLICKHOUSE_USER=admin
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}

  zookeeper:
    image: zookeeper:3.9
    restart: unless-stopped
    volumes:
      - zookeeper-data:/data
      - zookeeper-logs:/datalog

  kafka:
    image: bitnami/kafka:3.7
    restart: unless-stopped
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
    volumes:
      - kafka-data:/bitnami/kafka
    depends_on:
      - zookeeper

volumes:
  clickhouse-data:
  kafka-data:
  zookeeper-data:
  zookeeper-logs:

# Kiểm tra tất cả containers đang chạy
docker ps --format "table {{.Names}}\t{{.Status}}"

# Output mong đợi:
# NAMES                        STATUS
# docker-clickhouse-1          Up 2 minutes (healthy)
# docker-kafka-1               Up 2 minutes
# docker-signoz-frontend-1     Up 2 minutes
# docker-signoz-query-service-1 Up 2 minutes
# docker-zookeeper-1           Up 2 minutes
# docker-signoz-otel-collector-1 Up 2 minutes

# Test health endpoint
curl http://localhost:3301/api/v1/health
# Output: {"status":"ok"}

# Node.js —— zero code changes
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" \
OTEL_RESOURCE_ATTRIBUTES="service.name=payment-service" \
npx @opentelemetry/auto-instrumentations-node ./server.js

# Python —— zero code changes
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" \
OTEL_RESOURCE_ATTRIBUTES="service.name=order-service" \
opentelemetry-instrument python manage.py runserver

# Java —— thêm agent JVM flag
java -javaagent:opentelemetry-javaagent.jar \
  -Dotel.exporter.otlp.endpoint=http://localhost:4317 \
  -Dotel.resource.attributes=service.name=inventory-service \
  -jar application.jar

# Go —— thêm auto-instrumentation package
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4317" \
OTEL_RESOURCE_ATTRIBUTES="service.name=api-gateway" \
go run main.go

# Python Flask với manual instrumentation
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.resources import Resource, SERVICE_NAME
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from flask import Flask

# Cấu hình tracer
resource = Resource.create({SERVICE_NAME: "payment-service"})
provider = TracerProvider(resource=resource)
otlp_exporter = OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True)
processor = BatchSpanProcessor(otlp_exporter)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

app = Flask(__name__)
tracer = trace.get_tracer(__name__)

@app.route("/process-payment", methods=["POST"])
def process_payment():
    with tracer.start_as_current_span("process_payment") as span:
        span.set_attribute("payment.amount", 149.00)
        span.set_attribute("payment.currency", "USD")

        with tracer.start_as_current_span("validate_card"):
            # Card validation logic
            pass

        with tracer.start_as_current_span("charge_stripe"):
            # Stripe API call
            pass

        return {"status": "success"}

# Tạo dashboard tùy chỉnh qua API
curl -X POST http://localhost:3301/api/v1/dashboards \
  -H "Content-Type: application/json" \
  -d '{
    "title": "Payment Service Health",
    "description": "Key metrics for payment processing",
    "panels": [
      {
        "title": "Request Rate",
        "query": "SELECT count() FROM signoz_traces.signoz_index_v2 WHERE serviceName = '\''payment-service'\''",
        "widgetType": "graph",
        "timeRange": "1h"
      },
      {
        "title": "P95 Latency",
        "query": "SELECT histogramQuantile(0.95)(durationNano) / 1000000 FROM signoz_traces.signoz_index_v2 WHERE serviceName = '\''payment-service'\''",
        "widgetType": "value",
        "unit": "ms"
      },
      {
        "title": "Error Rate %",
        "query": "SELECT (countIf(statusCode = 2) * 100.0 / count()) FROM signoz_traces.signoz_index_v2 WHERE serviceName = '\''payment-service'\''",
        "widgetType": "gauge"
      }
    ]
  }'

# docker-compose.ha.yml — multi-node ClickHouse với ZooKeeper
services:
  clickhouse-1:
    image: clickhouse/clickhouse-server:24.3-alpine
    volumes:
      - clickhouse1-data:/var/lib/clickhouse
      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/cluster.xml
    environment:
      - CLICKHOUSE_USER=admin
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}

  clickhouse-2:
    image: clickhouse/clickhouse-server:24.3-alpine
    volumes:
      - clickhouse2-data:/var/lib/clickhouse
      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/cluster.xml
    environment:
      - CLICKHOUSE_USER=admin
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}

  clickhouse-3:
    image: clickhouse/clickhouse-server:24.3-alpine
    volumes:
      - clickhouse3-data:/var/lib/clickhouse
      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/cluster.xml
    environment:
      - CLICKHOUSE_USER=admin
      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}

  # Nginx load balancer cho ClickHouse
  clickhouse-lb:
    image: nginx:alpine
    volumes:
      - ./nginx-clickhouse.conf:/etc/nginx/nginx.conf
    ports:
      - "8123:8123"
      - "9000:9000"

# Cấu hình backup ClickHouse S3
<clickhouse>
  <storage_configuration>
    <disks>
      <s3_disk>
        <type>s3</type>
        <endpoint>https://s3.amazonaws.com/my-bucket/signoz/</endpoint>
        <access_key_id>${AWS_ACCESS_KEY}</access_key_id>
        <secret_access_key>${AWS_SECRET_KEY}</secret_access_key>
      </s3_disk>
    </disks>
    <policies>
      <tiered_storage>
        <volumes>
          <hot>
            <disk>default</disk>
            <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
          </hot>
          <cold>
            <disk>s3_disk</disk>
          </cold>
        </volumes>
      </tiered_storage>
    </policies>
  </storage_configuration>
</clickhouse>

# alert-rules.yml — SigNoz alert manager rules
groups:
  - name: payment_service_alerts
    rules:
      - alert: HighErrorRate
        expr: |
          (
            sum(rate(signoz_calls_total{service_name="payment-service",status_code="STATUS_CODE_ERROR"}[5m]))
            /
            sum(rate(signoz_calls_total{service_name="payment-service"}[5m]))
          ) > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Payment service error rate > 5%"
          description: "Error rate is {{ $value }}"

      - alert: HighP95Latency
        expr: histogramQuantile(0.95)(rate(signoz_latency_bucket{service_name="payment-service"}[5m])) > 500000000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Payment service P95 latency > 500ms"
          description: "P95 latency is {{ $value }}ns"

      - alert: LogErrorSpike
        expr: rate(signoz_logs_total{severity="ERROR"}[5m]) > 100
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Log error spike detected"
          description: "{{ $value }} errors/minute"

# signoz-otel-collector-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: signoz-otel-collector
  namespace: signoz
spec:
  ports:
    - name: otlp-grpc
      port: 4317
      protocol: TCP
    - name: otlp-http
      port: 4318
      protocol: TCP
  selector:
    app.kubernetes.io/name: otel-collector

---
# Instrument deployment bằng cách thêm OTel env vars
apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-service
spec:
  template:
    spec:
      containers:
        - name: payment-service
          image: payment-service:1.2.3
          env:
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: "http://signoz-otel-collector.signoz.svc.cluster.local:4317"
            - name: OTEL_RESOURCE_ATTRIBUTES
              value: "service.name=payment-service,service.namespace=production"
            - name: OTEL_TRACES_SAMPLER
              value: "parentbased_traceidratio"
            - name: OTEL_TRACES_SAMPLER_ARG
              value: "0.1"  # Sample 10% traces

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 1000
    policies:
      - name: errors
        type: status_code
        status_code: {status_codes: [ERROR]}
      - name: slow_requests
        type: latency
        latency: {threshold_ms: 500}
      - name: probabilistic
        type: probabilistic
        probabilistic: {sampling_percentage: 10}

exporters:
  clickhousetraces:
    datasource: tcp://clickhouse:9000
    database: signoz_traces

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [tail_sampling]
      exporters: [clickhousetraces]

receivers:
  prometheus:
    config:
      scrape_configs:
        - job_name: 'my-app'
          static_configs:
            - targets: ['my-app:9090']