VAST Data Integration Guide¶

export VAST_ACCESS_KEY="your-access-key"
export VAST_SECRET_KEY="your-secret-key"

response_cache:
  enabled: true

  # "tiered" enables the L1 + L2 architecture
  backend: tiered

  # Global TTL for all cache entries
  ttl: "24h"

  # Maximum response body size eligible for caching
  max_response_size: 1048576         # 1 MiB
  max_stream_buffer_size: 10485760   # 10 MiB

  # L1: Redis (hot cache — fast lookup, limited capacity)
  l1:
    type: memory   # or "redis" for distributed L1
    max_value_size: 1048576   # values larger than 1 MiB go directly to L2

  # Redis config used by L1 when l1.type is "redis"
  redis:
    url: "redis://redis-service:6379"
    pool_size: 16
    key_prefix: "cr:resp:"
    fallback_to_memory: true

  # L2: VAST Data S3 (warm cache — high capacity, durable)
  l2:
    type: s3
    endpoint: "https://vast-s3.example.com"
    bucket: "llm-response-cache"
    key_prefix: "response-cache/"
    region: "us-east-1"
    access_key: "${VAST_ACCESS_KEY}"
    secret_key: "${VAST_SECRET_KEY}"
    # Optional: override TTL for L2 entries (default: inherits global ttl)
    ttl_override: "7d"

  # Tiered cache promotion behavior
  tiered:
    promote_on_hit: true       # Promote L2 hits back to L1
    l1_promotion_ttl: "30m"    # TTL for promoted L1 entries

kv_cache_index:
  enabled: true
  backend: memory   # or "redis" for multi-instance deployments

  # Scale max_entries with the number of unique prompts × number of backends
  max_entries: 500000

  # Entries expire after 15 minutes; adjust based on vLLM eviction rate
  entry_ttl_seconds: 900

  scoring:
    overlap_weight: 0.6
    load_weight: 0.3
    health_weight: 0.1

    # Only activate KV-aware routing when a backend holds ≥30% coverage
    min_overlap_threshold: 0.30

    # GPU-resident data gets full overlap credit
    gpu_tier_weight: 1.0

    # Storage-offloaded data is valuable but incurs reload latency — discount it
    storage_tier_weight: 0.6

  # Track GPU hot vs. VAST warm tiers
  storage_offloading:
    enabled: true
    # When vLLM emits only cache_created/cache_evicted (no cache_offloaded),
    # treat evictions as offloads to VAST rather than permanent removal
    treat_eviction_as_offload: true

  # Subscribe to KV events from each vLLM backend
  event_sources:
    - backend_name: vllm-1
      endpoint: "http://vllm-1:8000/v1/kv_events"
      reconnect_interval_ms: 5000

    - backend_name: vllm-2
      endpoint: "http://vllm-2:8000/v1/kv_events"
      reconnect_interval_ms: 5000

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --max-model-len 32768 \
  --kv-transfer-config '{"kv_connector":"VastKVConnector","kv_buffer_device":"cpu","kv_buffer_size":4e9,"kv_role":"kv_both","kv_connector_extra_config":{"vast_endpoint":"http://vast-cluster:8080","kv_namespace":"inference/kv-cache"}}'

disaggregated_serving:
  enabled: true

  # Timeout for the prefill computation phase
  prefill_timeout: "60s"

  # Timeout for KV tensor transfer between VAST and workers
  kv_transfer_timeout: "15s"

  # Fall back to unified backends if disaggregated backends are unavailable
  fallback_to_unified: true

  # Default VAST storage for backends that do not specify their own
  default_external_storage:
    endpoint: "http://vast-cluster:8080"
    kv_namespace: "inference/kv-cache"
    # Credentials optional if VAST is configured for anonymous access
    # credentials: "${VAST_CREDENTIALS}"

backends:
  # Prefill workers — computes KV tensors for input prompts
  - name: prefill-worker-1
    url: "http://vllm-prefill-1:8000"
    role: prefill
    external_storage:
      endpoint: "http://vast-cluster:8080"
      kv_namespace: "inference/kv-cache"

  - name: prefill-worker-2
    url: "http://vllm-prefill-2:8000"
    role: prefill
    # Inherits default_external_storage when external_storage is omitted

  # Decode workers — generates tokens from cached KV data
  - name: decode-worker-1
    url: "http://vllm-decode-1:8000"
    role: decode
    weight: 2

  - name: decode-worker-2
    url: "http://vllm-decode-2:8000"
    role: decode
    weight: 2

  - name: decode-worker-3
    url: "http://vllm-decode-3:8000"
    role: decode
    weight: 2

  # Unified fallback — handles both phases when disaggregated pool is unavailable
  - name: unified-fallback
    url: "http://vllm-unified:8000"
    role: unified

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --kv-transfer-config '{"kv_connector":"VastKVConnector","kv_role":"kv_producer","kv_connector_extra_config":{"vast_endpoint":"http://vast-cluster:8080","kv_namespace":"inference/kv-cache"}}'

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --kv-transfer-config '{"kv_connector":"VastKVConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"vast_endpoint":"http://vast-cluster:8080","kv_namespace":"inference/kv-cache"}}'

server:
  host: "0.0.0.0"
  port: 8080

selection_strategy: LeastLatency

# ─── Prefix-Aware Routing (Tier 1) ───────────────────────────────────────────
prefix_routing:
  enabled: true
  max_prefix_length: 2048
  load_factor_epsilon: 0.20
  virtual_nodes: 200

# ─── Tiered Response Cache (Tier 2 + VAST S3 L2) ─────────────────────────────
response_cache:
  enabled: true
  backend: tiered
  ttl: "24h"
  max_response_size: 2097152       # 2 MiB
  max_stream_buffer_size: 20971520 # 20 MiB

  # L1: Redis (hot, fast, limited capacity)
  l1:
    type: redis
    max_value_size: 524288   # Values >512 KiB go directly to L2

  redis:
    url: "redis://redis-cluster:6379"
    pool_size: 32
    key_prefix: "cr:resp:"
    connect_timeout_ms: 3000
    command_timeout_ms: 1000
    fallback_to_memory: true

  # L2: VAST Data S3 (warm, durable, high capacity)
  l2:
    type: s3
    endpoint: "https://vast-s3.prod.example.com"
    bucket: "prod-llm-response-cache"
    key_prefix: "response-cache/"
    region: "us-east-1"
    access_key: "${VAST_ACCESS_KEY}"
    secret_key: "${VAST_SECRET_KEY}"
    ttl_override: "7d"

  tiered:
    promote_on_hit: true
    l1_promotion_ttl: "30m"

# ─── KV Cache Index with VAST Offloading Awareness (Tier 4) ──────────────────
kv_cache_index:
  enabled: true
  backend: redis   # Reuses the redis pool from response_cache.redis
  max_entries: 1000000
  entry_ttl_seconds: 1800

  scoring:
    overlap_weight: 0.6
    load_weight: 0.3
    health_weight: 0.1
    min_overlap_threshold: 0.25
    gpu_tier_weight: 1.0
    storage_tier_weight: 0.6

  storage_offloading:
    enabled: true
    treat_eviction_as_offload: true

  event_sources:
    - backend_name: prefill-worker-1
      endpoint: "http://vllm-prefill-1:8000/v1/kv_events"
      reconnect_interval_ms: 5000
    - backend_name: prefill-worker-2
      endpoint: "http://vllm-prefill-2:8000/v1/kv_events"
      reconnect_interval_ms: 5000
    - backend_name: decode-worker-1
      endpoint: "http://vllm-decode-1:8000/v1/kv_events"
      reconnect_interval_ms: 5000
    - backend_name: decode-worker-2
      endpoint: "http://vllm-decode-2:8000/v1/kv_events"
      reconnect_interval_ms: 5000
    - backend_name: decode-worker-3
      endpoint: "http://vllm-decode-3:8000/v1/kv_events"
      reconnect_interval_ms: 5000

# ─── Disaggregated Prefill/Decode Serving ────────────────────────────────────
disaggregated_serving:
  enabled: true
  prefill_timeout: "60s"
  kv_transfer_timeout: "15s"
  fallback_to_unified: true
  default_external_storage:
    endpoint: "http://vast-cluster.prod.example.com:8080"
    kv_namespace: "prod/inference/kv-cache"

# ─── Backends ─────────────────────────────────────────────────────────────────
backends:
  - name: prefill-worker-1
    url: "http://vllm-prefill-1:8000"
    role: prefill
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: prefill-worker-2
    url: "http://vllm-prefill-2:8000"
    role: prefill
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: decode-worker-1
    url: "http://vllm-decode-1:8000"
    role: decode
    weight: 2
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: decode-worker-2
    url: "http://vllm-decode-2:8000"
    role: decode
    weight: 2
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: decode-worker-3
    url: "http://vllm-decode-3:8000"
    role: decode
    weight: 2
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

  - name: unified-fallback
    url: "http://vllm-unified:8000"
    role: unified
    models: ["meta-llama/Llama-3.1-70B-Instruct"]

# ─── Health Checks ────────────────────────────────────────────────────────────
health_checks:
  interval: "15s"
  timeout: "5s"
  unhealthy_threshold: 3
  healthy_threshold: 2

# ─── Circuit Breaker ──────────────────────────────────────────────────────────
circuit_breaker:
  enabled: true
  failure_threshold: 5
  recovery_timeout: "30s"

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 4 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --max-model-len 32768 \
  --gpu-memory-utilization 0.85 \
  --kv-transfer-config '{
    "kv_connector": "VastKVConnector",
    "kv_role": "kv_producer",
    "kv_buffer_device": "cpu",
    "kv_buffer_size": 8e9,
    "kv_connector_extra_config": {
      "vast_endpoint": "http://vast-cluster.prod.example.com:8080",
      "kv_namespace": "prod/inference/kv-cache"
    }
  }'

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 4 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --max-model-len 32768 \
  --gpu-memory-utilization 0.90 \
  --kv-transfer-config '{
    "kv_connector": "VastKVConnector",
    "kv_role": "kv_consumer",
    "kv_buffer_device": "cpu",
    "kv_buffer_size": 8e9,
    "kv_connector_extra_config": {
      "vast_endpoint": "http://vast-cluster.prod.example.com:8080",
      "kv_namespace": "prod/inference/kv-cache"
    }
  }'

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --host 0.0.0.0 \
  --port 8000 \
  --tensor-parallel-size 4 \
  --enable-prefix-caching \
  --kv-cache-dtype auto \
  --max-model-len 32768