VAST Data 통합 가이드¶

export VAST_ACCESS_KEY="your-access-key"
export VAST_SECRET_KEY="your-secret-key"

response_cache:
  enabled: true

  # "tiered"로 L1 + L2 아키텍처 활성화
  backend: tiered

  # 모든 캐시 항목의 전역 TTL
  ttl: "24h"

  # 캐시 대상 최대 응답 본문 크기
  max_response_size: 1048576         # 1 MiB
  max_stream_buffer_size: 10485760   # 10 MiB

  # L1: Redis (핫 캐시 — 빠른 조회, 제한된 용량)
  l1:
    type: memory   # 분산 L1에는 "redis" 사용
    max_value_size: 1048576   # 1 MiB 초과 값은 L2로 직접 저장

  # l1.type이 "redis"일 때 L1에 사용되는 Redis 설정
  redis:
    url: "redis://redis-service:6379"
    pool_size: 16
    key_prefix: "cr:resp:"
    fallback_to_memory: true

  # L2: VAST Data S3 (웜 캐시 — 고용량, 내구성)
  l2:
    type: s3
    endpoint: "https://vast-s3.example.com"
    bucket: "llm-response-cache"
    key_prefix: "response-cache/"
    region: "us-east-1"
    access_key: "${S3_ACCESS_KEY}"
    secret_key: "${S3_SECRET_KEY}"
    # 선택 사항: L2 항목에 대한 TTL 재정의 (기본값: 전역 ttl 상속)
    ttl_override: "7d"

  # 계층형 캐시 승격 동작
  tiered:
    promote_on_hit: true       # L2 적중을 L1으로 승격
    l1_promotion_ttl: "30m"    # 승격된 L1 항목의 TTL

kv_cache_index:
  enabled: true
  backend: memory   # 멀티 인스턴스 배포에는 "redis" 사용

  # 고유 프롬프트 수 × 백엔드 수에 맞게 max_entries 조정
  max_entries: 500000

  # 항목은 15분 후 만료; vLLM 축출 속도에 따라 조정
  entry_ttl_seconds: 900

  scoring:
    overlap_weight: 0.6
    load_weight: 0.3
    health_weight: 0.1

    # 백엔드가 ≥30% 커버리지를 보유할 때만 KV 인식 라우팅 활성화
    min_overlap_threshold: 0.30

    # GPU 상주 데이터는 전체 overlap 점수 부여
    gpu_tier_weight: 1.0

    # 스토리지 오프로드 데이터는 가치 있지만 재로드 지연 발생 — 할인 적용
    storage_tier_weight: 0.6

  # GPU 핫 vs. VAST 웜 티어 추적
  storage_offloading:
    enabled: true
    # vLLM이 cache_created/cache_evicted만 내보낼 때 (cache_offloaded 없음),
    # 축출을 영구 제거가 아닌 VAST 오프로드로 처리
    treat_eviction_as_offload: true

  # 각 vLLM 백엔드에서 KV 이벤트 구독
  event_sources:
    - backend_name: vllm-1
      endpoint: "http://vllm-1:8000/v1/kv_events"
      reconnect_interval_ms: 5000

    - backend_name: vllm-2
      endpoint: "http://vllm-2:8000/v1/kv_events"
      reconnect_interval_ms: 5000

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --max-model-len 32768 \
  --kv-transfer-config '{"kv_connector":"VastKVConnector","kv_buffer_device":"cpu","kv_buffer_size":4e9,"kv_role":"kv_both","kv_connector_extra_config":{"vast_endpoint":"http://vast-cluster:8080","kv_namespace":"inference/kv-cache"}}'

disaggregated_serving:
  enabled: true

  # prefill 연산 단계 타임아웃
  prefill_timeout: "60s"

  # VAST와 워커 간 KV 텐서 전송 타임아웃
  kv_transfer_timeout: "15s"

  # 분리형 백엔드 사용 불가 시 통합형 백엔드로 폴백
  fallback_to_unified: true

  # 자체 external_storage를 지정하지 않은 백엔드의 기본 외부 스토리지
  default_external_storage:
    endpoint: "http://vast-cluster:8080"
    kv_namespace: "inference/kv-cache"
    # VAST가 익명 접근으로 설정된 경우 자격 증명 선택 사항
    # credentials: "${VAST_CREDENTIALS}"

backends:
  # Prefill 워커 — 입력 프롬프트의 KV 텐서 계산
  - name: prefill-worker-1
    url: "http://vllm-prefill-1:8000"
    role: prefill
    external_storage:
      endpoint: "http://vast-cluster:8080"
      kv_namespace: "inference/kv-cache"

  - name: prefill-worker-2
    url: "http://vllm-prefill-2:8000"
    role: prefill
    # external_storage 생략 시 default_external_storage 상속

  # Decode 워커 — 캐시된 KV 데이터로 토큰 생성
  - name: decode-worker-1
    url: "http://vllm-decode-1:8000"
    role: decode
    weight: 2

  - name: decode-worker-2
    url: "http://vllm-decode-2:8000"
    role: decode
    weight: 2

  - name: decode-worker-3
    url: "http://vllm-decode-3:8000"
    role: decode
    weight: 2

  # 통합형 폴백 — 분리형 풀 사용 불가 시 두 단계 모두 처리
  - name: unified-fallback
    url: "http://vllm-unified:8000"
    role: unified

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --kv-transfer-config '{"kv_connector":"VastKVConnector","kv_role":"kv_producer","kv_connector_extra_config":{"vast_endpoint":"http://vast-cluster:8080","kv_namespace":"inference/kv-cache"}}'

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --kv-transfer-config '{"kv_connector":"VastKVConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"vast_endpoint":"http://vast-cluster:8080","kv_namespace":"inference/kv-cache"}}'

server:
  host: "0.0.0.0"
  port: 8080

selection_strategy: LeastLatency

# ─── Prefix 인식 라우팅 (티어 1) ─────────────────────────────────────────────
prefix_routing:
  enabled: true
  max_prefix_length: 2048
  load_factor_epsilon: 0.20
  virtual_nodes: 200

# ─── 계층형 응답 캐시 (티어 2 + VAST S3 L2) ──────────────────────────────────
response_cache:
  enabled: true
  backend: tiered
  ttl: "24h"
  max_response_size: 2097152       # 2 MiB
  max_stream_buffer_size: 20971520 # 20 MiB

  # L1: Redis (핫, 빠름, 제한된 용량)
  l1:
    type: redis
    max_value_size: 524288   # 512 KiB 초과 값은 L2로 직접 저장

  redis:
    url: "redis://redis-cluster:6379"
    pool_size: 32
    key_prefix: "cr:resp:"
    connect_timeout_ms: 3000
    command_timeout_ms: 1000
    fallback_to_memory: true

  # L2: VAST Data S3 (웜, 내구성, 고용량)
  l2:
    type: s3
    endpoint: "https://vast-s3.prod.example.com"
    bucket: "prod-llm-response-cache"
    key_prefix: "response-cache/"
    region: "us-east-1"
    access_key: "${S3_ACCESS_KEY}"
    secret_key: "${S3_SECRET_KEY}"
    ttl_override: "7d"

  tiered:
    promote_on_hit: true
    l1_promotion_ttl: "30m"

# ─── VAST 오프로딩 인식 KV 캐시 인덱스 (티어 4) ─────────────────────────────
kv_cache_index:
  enabled: true
  backend: redis   # response_cache.redis의 redis 풀 재사용
  max_entries: 1000000
  entry_ttl_seconds: 1800

  scoring:
    overlap_weight: 0.6
    load_weight: 0.3
    health_weight: 0.1
    min_overlap_threshold: 0.25
    gpu_tier_weight: 1.0
    storage_tier_weight: 0.6

  storage_offloading:
    enabled: true
    treat_eviction_as_offload: true

  event_sources:
    - backend_name: prefill-worker-1
      endpoint: "http://vllm-prefill-1:8000/v1/kv_events"
      reconnect_interval_ms: 5000
    - backend_name: prefill-worker-2
      endpoint: "http://vllm-prefill-2:8000/v1/kv_events"
      reconnect_interval_ms: 5000
    - backend_name: decode-worker-1
      endpoint: "http://vllm-decode-1:8000/v1/kv_events"
      reconnect_interval_ms: 5000
    - backend_name: decode-worker-2
      endpoint: "http://vllm-decode-2:8000/v1/kv_events"
      reconnect_interval_ms: 5000
    - backend_name: decode-worker-3
      endpoint: "http://vllm-decode-3:8000/v1/kv_events"
      reconnect_interval_ms: 5000

# ─── 분리형 Prefill/Decode 서빙 ──────────────────────────────────────────────
disaggregated_serving:
  enabled: true
  prefill_timeout: "60s"
  kv_transfer_timeout: "15s"
  fallback_to_unified: true
  default_external_storage:
    endpoint: "http://vast-cluster.prod.example.com:8080"
    kv_namespace: "prod/inference/kv-cache"

# ─── 백엔드 ───────────────────────────────────────────────────────────────────
backends:
  - name: prefill-worker-1
    url: "http://vllm-prefill-1:8000"
    role: prefill
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: prefill-worker-2
    url: "http://vllm-prefill-2:8000"
    role: prefill
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: decode-worker-1
    url: "http://vllm-decode-1:8000"
    role: decode
    weight: 2
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: decode-worker-2
    url: "http://vllm-decode-2:8000"
    role: decode
    weight: 2
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: decode-worker-3
    url: "http://vllm-decode-3:8000"
    role: decode
    weight: 2
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: unified-fallback
    url: "http://vllm-unified:8000"
    role: unified
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

# ─── 헬스 체크 ────────────────────────────────────────────────────────────────
health_checks:
  interval: "15s"
  timeout: "5s"
  unhealthy_threshold: 3
  healthy_threshold: 2

# ─── 서킷 브레이커 ────────────────────────────────────────────────────────────
circuit_breaker:
  enabled: true
  failure_threshold: 5
  recovery_timeout: "30s"

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 4 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --max-model-len 32768 \
  --gpu-memory-utilization 0.85 \
  --kv-transfer-config '{
    "kv_connector": "VastKVConnector",
    "kv_role": "kv_producer",
    "kv_buffer_device": "cpu",
    "kv_buffer_size": 8e9,
    "kv_connector_extra_config": {
      "vast_endpoint": "http://vast-cluster.prod.example.com:8080",
      "kv_namespace": "prod/inference/kv-cache"
    }
  }'

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 4 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --max-model-len 32768 \
  --gpu-memory-utilization 0.90 \
  --kv-transfer-config '{
    "kv_connector": "VastKVConnector",
    "kv_role": "kv_consumer",
    "kv_buffer_device": "cpu",
    "kv_buffer_size": 8e9,
    "kv_connector_extra_config": {
      "vast_endpoint": "http://vast-cluster.prod.example.com:8080",
      "kv_namespace": "prod/inference/kv-cache"
    }
  }'

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 4 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --max-model-len 32768