메트릭 및 모니터링¶

# 메트릭 보기
curl http://localhost:8000/metrics

scrape_configs:
    - job_name: 'continuum-router'
    static_configs:
      - targets: ['localhost:8000']
    scrape_interval: 15s

metrics:
  # 메트릭 수집 활성화/비활성화
  enabled: true

  # 메트릭 엔드포인트 경로
  endpoint: "/metrics"

  # 메트릭 폭발을 방지하기 위한 카디널리티 제한
  cardinality_limit:
    max_labels_per_metric: 100
    max_unique_label_values: 1000

  # 선택적 메트릭 (성능을 위해 기본적으로 비활성화)
  optional_metrics:
    enable_request_body_size: false
    enable_response_body_size: false
    enable_detailed_errors: true

# 메트릭 활성화/비활성화
METRICS_ENABLED=true

# 메트릭 엔드포인트 변경
METRICS_ENDPOINT=/custom/metrics

# 선택적 메트릭 활성화
METRICS_ENABLE_BODY_SIZE=true

global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
    - job_name: 'continuum-router'
    static_configs:
      - targets: ['router1:8000', 'router2:8000']
    metric_relabel_configs:
      # 필요시 높은 카디널리티 메트릭 삭제
      - source_labels: [__name__]
        regex: 'http_request_duration_seconds_bucket'
        action: drop

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: continuum-router
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: continuum-router
  endpoints:
    - port: metrics
    interval: 15s
    path: /metrics

- alert: BackendDown
  expr: backend_health_status == 0
  for: 1m
  annotations:
    summary: "백엔드 {{ $labels.backend_id }}이(가) 다운됨"

- alert: HighErrorRate
  expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05
  for: 5m
  annotations:
    summary: "높은 오류율: {{ $value | humanizePercentage }}"

- alert: HighLatency
  expr: histogram_quantile(0.95, http_request_duration_seconds) > 1
  for: 5m
  annotations:
    summary: "P95 지연 시간 1초 초과: {{ $value | humanizeDuration }}"

- alert: LowCacheHitRate
  expr: rate(model_cache_hits_total[5m]) / rate(model_cache_total[5m]) < 0.8
  for: 10m
  annotations:
    summary: "캐시 적중률 80% 미만: {{ $value | humanizePercentage }}"

sum(rate(http_requests_total[5m])) by (status)

histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (endpoint, le)
)

sum(backend_health_status) by (backend_id)

topk(10, sum(rate(model_usage_total[1h])) by (model))

sum(rate(http_requests_total{status=~"5.."}[5m])) /
sum(rate(http_requests_total[5m])) * 100

import requests
from prometheus_client.parser import text_string_to_metric_families

# 메트릭 가져오기
response = requests.get('http://localhost:8000/metrics')
metrics = text_string_to_metric_families(response.text)

# 메트릭 처리
for family in metrics:
    for sample in family.samples:
        if sample.name == 'http_requests_total':
            print(f"엔드포인트: {sample.labels['endpoint']}, 카운트: {sample.value}")

#!/bin/bash
# 30초마다 메트릭을 수집하고 파일에 저장

while true; do
  timestamp=$(date +%s)
  curl -s http://localhost:8000/metrics > "metrics_${timestamp}.txt"
  sleep 30
done

# 좋음: 낮은 카디널리티
labels:
  status: "200"  # ~5개 가능한 값
  method: "GET"  # ~7개 가능한 값

# 나쁨: 높은 카디널리티
labels:
  user_id: "12345"  # 무제한
  request_id: "abc-123"  # 요청당 고유