Portkey AI Gateway 2026: 200+ 모델을 관리하는 LLM 게이트웨이와 관찰 가능성 — 프로덕션 설정

# 저장소 복제
git clone https://github.com/Portkey-AI/gateway.git
cd gateway

# Docker로 실행
docker run -p 8787:8787 -e PORTKEY_GATEWAY_API_KEY=your-gateway-key portkeyai/gateway:latest

version: '3.8'
services:
  portkey-gateway:
    image: portkeyai/gateway:latest
    ports:
      - "8787:8787"
    environment:
      - PORTKEY_GATEWAY_API_KEY=${GATEWAY_API_KEY}
      - CACHE_ENABLED=true
      - CACHE_TTL=3600
    volumes:
      - ./config:/app/config
    restart: unless-stopped

apiVersion: apps/v1
kind: Deployment
metadata:
  name: portkey-gateway
spec:
  replicas: 3
  selector:
    matchLabels:
      app: portkey-gateway
  template:
    metadata:
      labels:
        app: portkey-gateway
    spec:
      containers:
      - name: gateway
        image: portkeyai/gateway:latest
        ports:
        - containerPort: 8787
        env:
        - name: PORTKEY_GATEWAY_API_KEY
          valueFrom:
            secretKeyRef:
              name: portkey-secrets
              key: gateway-api-key
---
apiVersion: v1
kind: Service
metadata:
  name: portkey-gateway-service
spec:
  selector:
    app: portkey-gateway
  ports:
  - port: 80
    targetPort: 8787
  type: ClusterIP

providers:
  openai-primary:
    type: openai
    api_key: ${OPENAI_API_KEY}
    organization: ${OPENAI_ORG_ID}
    
  anthropic-primary:
    type: anthropic
    api_key: ${ANTHROPIC_API_KEY}
    
  azure-gpt4:
    type: azure-openai
    api_key: ${AZURE_API_KEY}
    resource_name: ${AZURE_RESOURCE_NAME}
    deployment_id: gpt-4
    api_version: 2025-12-01
    
  google-gemini:
    type: google
    api_key: ${GOOGLE_API_KEY}
    
  mistral-local:
    type: mistral
    api_key: ${MISTRAL_API_KEY}
    base_url: http://mistral-service:8000/v1

# 환경 변수 설정
export OPENAI_API_KEY="sk-..."
export ANTHROPIC_API_KEY="sk-ant-..."
export GATEWAY_API_KEY="pk-..."

# 구성과 함께 게이트웨이 시작
docker run -p 8787:8787 \
  -e PORTKEY_GATEWAY_API_KEY=$GATEWAY_API_KEY \
  -v $(pwd)/providers.yaml:/app/config/providers.yaml \
  portkeyai/gateway:latest

curl -X POST http://localhost:8787/v1/chat/completions \
  -H "Authorization: Bearer ${GATEWAY_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gpt-4o",
    "provider": "openai-primary",
    "messages": [
      {"role": "system", "content": "당신은 유용한 어시스턴트입니다."},
      {"role": "user", "content": "양자 컴퓨팅을 간단한 용어로 설명하세요."}
    ]
  }'

# 동일한 요청, 다른 제공업체 — model/provider 필드만 변경
curl -X POST http://localhost:8787/v1/chat/completions \
  -H "Authorization: Bearer ${GATEWAY_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "claude-sonnet-4",
    "provider": "anthropic-primary",
    "messages": [
      {"role": "user", "content": "양자 컴퓨팅을 간단한 용어로 설명하세요."}
    ],
    "max_tokens": 1024
  }'

from portkey_ai import Portkey

# 클라이언트 초기화
portkey = Portkey(
    api_key="your-gateway-api-key",
    virtual_key="openai-primary"  # 구성된 제공업체 참조
)

# 대화 완성
response = portkey.chat.completions.create(
    model="gpt-4o",
    messages=[
        {"role": "system", "content": "당신은 유용한 어시스턴트입니다."},
        {"role": "user", "content": "피볂치 수를 계산하는 Python 함수를 작성하세요."}
    ]
)
print(response.choices[0].message.content)

import portkey_ai

portkey = portkey_ai.Portkey(api_key="your-gateway-api-key")

stream = portkey.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "AI에 대한 시를 써주세요."}],
    stream=True
)

for chunk in stream:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="")

# config/load-balance.yaml
strategies:
  gpt4-pool:
    type: load_balance
    providers:
      - provider: openai-primary
        weight: 1
      - provider: azure-gpt4
        weight: 1
      - provider: openai-backup
        weight: 1

# 부하 분산 풀 사용
response = portkey.chat.completions.create(
    model="gpt-4o",
    config="gpt4-pool",  # 전략 참조
    messages=[{"role": "user", "content": "안녕하세요!"}]
)

strategies:
  production-fallback:
    type: fallback
    targets:
      - provider: azure-gpt4
        timeout: 10
        retry: 2
      - provider: openai-primary
        timeout: 15
        retry: 1
      - provider: anthropic-primary
        model: claude-sonnet-4
        timeout: 15
      - provider: google-gemini
        model: gemini-2.5-pro
        timeout: 20

# 게이트웨이는 성공할 때까지 순서대로 각 대상을 시도합니다
curl -X POST http://localhost:8787/v1/chat/completions \
  -H "Authorization: Bearer ${GATEWAY_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{
    "config": "production-fallback",
    "messages": [{"role": "user", "content": "중요한 비즈니스 질문"}]
  }'

strategies:
  smart-router:
    type: conditional
    rules:
      - condition: "request.messages[0].content.length > 4000"
        target: 
          provider: anthropic-primary
          model: claude-sonnet-4  # 더 나은 긴 컨텍스트 처리
      - condition: "request.user == 'code-assistant'"
        target:
          provider: openai-primary
          model: gpt-4o
      - condition: "default"
        target:
          provider: azure-gpt4
          model: gpt-4o-mini

cache:
  enabled: true
  mode: semantic  # 또는 "exact"는 정확한 일치 캐싱용
  ttl: 3600       # 캐시 생존 시간(초)
  max_size: 10000 # 최대 캐시 항목 수
  similarity_threshold: 0.95  # 의미론적 캐싱용

# 첫 번째 호출은 제공업체를 타고 결과를 캐싱합니다
response1 = portkey.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Kubernetes란 무엇인가요?"}],
    cache=True
)

# 유사한 호출은 즉시 캐시된 결과를 반환합니다
response2 = portkey.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Kubernetes에 대해 설명해주세요"}],
    cache=True
)

# 캐시 메트릭 확인
curl http://localhost:8787/v1/admin/cache/stats \
  -H "Authorization: Bearer ${GATEWAY_API_KEY}"

# 특정 캐시 항목 무효화
curl -X POST http://localhost:8787/v1/admin/cache/invalidate \
  -H "Authorization: Bearer ${GATEWAY_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{
    "pattern": "kubernetes",
    "provider": "openai-primary"
  }'

import portkey_ai

portkey = portkey_ai.Portkey(
    api_key="your-gateway-api-key",
    metadata={
        "user_id": "user-12345",
        "project": "customer-support-bot",
        "environment": "production",
        "team": "ai-platform"
    }
)

response = portkey.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "내 주문을 도와주세요"}]
)

# 응답에서 비용 정보에 액세스
print(f"입력 토큰: {response.usage.prompt_tokens}")
print(f"출력 토큰: {response.usage.completion_tokens}")
print(f"총 비용: ${response.usage.estimated_cost}")

# 제공업첼별 지출 보고서 가져오기
curl "http://localhost:8787/v1/admin/analytics/spend?start_date=2026-05-01&end_date=2026-05-19&group_by=provider" \
  -H "Authorization: Bearer ${GATEWAY_API_KEY}"

# 사용자별 지출 보고서 가져오기
curl "http://localhost:8787/v1/admin/analytics/spend?start_date=2026-05-01&end_date=2026-05-19&group_by=user_id" \
  -H "Authorization: Bearer ${GATEWAY_API_KEY}"

alerts:
  daily-budget:
    type: budget
    threshold: 500  # USD
    period: daily
    channels:
      - type: webhook
        url: https://hooks.slack.com/services/YOUR/WEBHOOK/URL
      - type: email
        address: team@company.com
  
  abnormal-spike:
    type: anomaly
    baseline_multiplier: 3
    window: 1h
    channels:
      - type: pagerduty
        integration_key: your-pd-key

from portkey_ai import Portkey

portkey = Portkey(api_key="your-gateway-api-key")

# 프롬프트 버전 배포
prompt = portkey.prompts.deploy(
    name="customer-support-classifier",
    version="1.2.0",
    prompt=[
        {"role": "system", "content": "당신은 지원 티켓 분류기입니다. 티켓을 다음으로 분류하세요: 결제, 기술, 기능 요청 또는 불만."},
        {"role": "user", "content": "티켓: {{ticket_content}}"}
    ],
    model="gpt-4o-mini",
    parameters={
        "temperature": 0.2,
        "max_tokens": 50
    }
)

# 관리형 프롬프트 렌더링 및 실행
response = portkey.prompts.render(
    name="customer-support-classifier",
    variables={
        "ticket_content": "이번 달에 구독이 두 번 청구되었습니다."
    }
)

print(response.choices[0].message.content)
# 출력: "결제"

# 프롬프트 버전 간 A/B 테스트 실행
response = portkey.prompts.render(
    name="customer-support-classifier",
    version="1.2.0",  # 50% 트래픽
    test_version="1.3.0-beta",  # 50% 트래픽
    variables={"ticket_content": "사진 업로드 시 앱이 충돌합니다"}
)

guardrails:
  input-validation:
    - type: keyword_filter
      blocklist: ["password", "ssn", "credit_card", "secret_key"]
      action: block
    - type: pii_detector
      entities: ["email", "phone", "ssn"]
      action: mask
    - type: toxicity_check
      threshold: 0.8
      action: block
      
  output-validation:
    - type: content_policy
      categories: ["hate", "violence", "self-harm"]
      action: block
    - type: response_format
      required_schema:
        type: json_object
      action: retry

# 요청에 가드레일 적용
response = portkey.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "사용자 입력 내용"}],
    guardrails=["input-validation", "output-validation"]
)

from portkey_ai import Portkey
import json

portkey = Portkey(api_key="your-gateway-api-key")

def custom_validator(request, response):
    """사용자 정의 비즈니스 로직 검증."""
    try:
        data = json.loads(response.choices[0].message.content)
        if "confidence" not in data or data["confidence"] < 0.7:
            return False, "신뢰도 점수가 너무 낮습니다"
        return True, None
    except json.JSONDecodeError:
        return False, "응답은 유효한 JSON이어야 합니다"

portkey.guardrails.register("confidence-check", custom_validator)

# 최근 요청 로그 쿼리
curl "http://localhost:8787/v1/admin/logs?limit=100&status=error" \
  -H "Authorization: Bearer ${GATEWAY_API_KEY}"

# 요청별 상세 로깅 활성화
response = portkey.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "안녕하세요"}],
    metadata={
        "trace_id": "trace-abc-123",
        "session_id": "session-xyz-789",
        "user_id": "user-456"
    }
)

observability:
  tracing:
    enabled: true
    exporter: otlp
    endpoint: http://jaeger-collector:4317
  metrics:
    enabled: true
    exporter: prometheus
    port: 9090

# 메트릭 스크래핑
curl http://localhost:8787/metrics

{
  "dashboard": {
    "title": "Portkey AI Gateway 개요",
    "panels": [
      {
        "title": "초당 요청 수",
        "targets": [
          {
            "expr": "rate(portkey_requests_total[5m])"
          }
        ]
      },
      {
        "title": "P95 지연 시간",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, portkey_request_duration_seconds_bucket)"
          }
        ]
      },
      {
        "title": "토큰 사용량",
        "targets": [
          {
            "expr": "sum by (model) (portkey_tokens_total)"
          }
        ]
      }
    ]
  }
}

# Redis 캐싱 및 PostgreSQL 로그가 있는 프로덕션 docker-compose
version: '3.8'
services:
  gateway:
    image: portkeyai/gateway:latest
    ports:
      - "8787:8787"
    environment:
      - PORTKEY_GATEWAY_API_KEY=${GATEWAY_API_KEY}
      - REDIS_URL=redis://redis:6379
      - DATABASE_URL=postgres://user:pass@postgres:5432/portkey
    depends_on:
      - redis
      - postgres
    deploy:
      replicas: 3
      resources:
        limits:
          memory: 2G
          cpus: '1.0'
  
  redis:
    image: redis:7-alpine
    volumes:
      - redis-data:/data
  
  postgres:
    image: postgres:16-alpine
    environment:
      POSTGRES_DB: portkey
      POSTGRES_USER: user
      POSTGRES_PASSWORD: ${DB_PASSWORD}
    volumes:
      - postgres-data:/var/lib/postgresql/data

volumes:
  redis-data:
  postgres-data:

# 게이트웨이 헬스 엔드포인트
curl http://localhost:8787/health

# 예상 응답
{"status": "healthy", "version": "2.5.0", "uptime": 86400}

# Kubernetes 활성 및 준비 프로브
livenessProbe:
  httpGet:
    path: /health
    port: 8787
  initialDelaySeconds: 10
  periodSeconds: 15

readinessProbe:
  httpGet:
    path: /ready
    port: 8787
  initialDelaySeconds: 5
  periodSeconds: 5

import openai

client = openai.OpenAI(
    api_key="your-portkey-gateway-key",
    base_url="http://localhost:8787/v1"
)

# 기존 코드가 변경 없이 작동합니다
response = client.chat.completions.create(...)