Configuration Examples¶

# Enterprise multi-backend configuration
server:
  bind_address: "0.0.0.0:8080"
  workers: 8
  connection_pool_size: 400

backends:
  # Primary OpenAI GPT models
  - name: "openai-primary"
    url: "https://api.openai.com"
    weight: 5
    models: ["gpt-4", "gpt-3.5-turbo"]
    retry_override:
      max_attempts: 3
      initial_delay: "500ms"
      max_delay: "10s"
      backoff_multiplier: 2.0
      jitter: true
      retryable_status_codes: [429, 502, 503, 504]
      retryable_errors: [ConnectionError, TimeoutError]
      timeout: "30s"

  # Secondary Azure OpenAI
  - name: "azure-openai"
    url: "https://your-resource.openai.azure.com"
    weight: 3
    models: ["gpt-4", "gpt-35-turbo"]

  # Local Ollama for open models
  - name: "local-ollama"
    url: "http://ollama:11434"
    weight: 2
    models: ["llama2", "mistral", "codellama"]

  # vLLM deployment
  - name: "vllm-cluster"
    url: "http://vllm-service:8000"
    weight: 4
    models: ["meta-llama/Llama-2-7b-chat-hf"]

health_checks:
  interval: "45s"
  timeout: "15s"
  unhealthy_threshold: 3
  healthy_threshold: 2
  endpoint: "/health"


retry:
  max_attempts: 4
  initial_delay: "100ms"
  max_delay: "10s"
  backoff_multiplier: 2.0
  jitter: true
  retryable_status_codes: [429, 502, 503, 504]
  retryable_errors: [ConnectionError, TimeoutError]
  timeout: "30s"

logging:
  level: "info"
  format: "json"

# Optimized for high-throughput scenarios
server:
  bind_address: "0.0.0.0:8080"
  workers: 16                     # High worker count
  connection_pool_size: 1000      # Large connection pool

backends:
  - name: "fast-backend-1"
    url: "http://backend1:8000"
    weight: 1
  - name: "fast-backend-2"
    url: "http://backend2:8000"
    weight: 1
  - name: "fast-backend-3"
    url: "http://backend3:8000"
    weight: 1

health_checks:
  interval: "30s"
  timeout: "5s"                   # Fast timeout
  unhealthy_threshold: 2          # Fail fast
  healthy_threshold: 1            # Recover quickly
  endpoint: "/health"


retry:
  max_attempts: 2                 # Fewer retries
  initial_delay: "50ms"           # Fast retries
  max_delay: "5s"
  backoff_multiplier: 2.0
  jitter: true
  retryable_status_codes: [429, 502, 503, 504]
  retryable_errors: [ConnectionError, TimeoutError]
  timeout: "15s"

logging:
  level: "warn"                   # Minimal logging for performance
  format: "json"

# Developer-friendly configuration
server:
  bind_address: "127.0.0.1:8080"  # Localhost only
  workers: 2                      # Fewer workers for development
  connection_pool_size: 20        # Small pool

backends:
  - name: "local-ollama"
    url: "http://localhost:11434"
    weight: 1

health_checks:
  interval: "10s"                 # Frequent checks for quick feedback
  timeout: "3s"
  unhealthy_threshold: 2
  healthy_threshold: 1
  endpoint: "/health"


retry:
  max_attempts: 1                 # Minimal retries for debugging
  initial_delay: "100ms"
  max_delay: "1s"
  backoff_multiplier: 2.0
  jitter: true
  retryable_status_codes: [429, 502, 503, 504]
  retryable_errors: [ConnectionError, TimeoutError]
  timeout: "5s"

logging:
  level: "debug"                  # Verbose logging
  format: "pretty"                # Human-readable