08. 监控菜谱：可验证的 Prometheus 规则、PromQL、Dashboard 面板

# TTFT p99，按 model 维度
histogram_quantile(0.99,
  sum by (model_name, le) (
    rate(vllm:time_to_first_token_seconds_bucket[5m])
  )
)

# TPOT p99
histogram_quantile(0.99,
  sum by (model_name, le) (
    rate(vllm:request_time_per_output_token_seconds_bucket[5m])
  )
)

# 错误率（按 finish_reason）
sum by (model_name) (rate(vllm:request_success_total{finished_reason="abort"}[5m]))
/
sum by (model_name) (rate(vllm:request_success_total[5m]))

# 队列深度 p95（waiting 数的 5min p95）
quantile_over_time(0.95, vllm:num_requests_waiting[5m])

# KV cache 利用率（瞬时）
vllm:kv_cache_usage_perc

# 抢占速率（每秒抢占次数）
rate(vllm:num_preemptions_total[5m])

# 每 step 平均 token 数（GPU 利用率代理）
histogram_quantile(0.5,
  sum by (le) (rate(vllm:iteration_tokens_total_bucket[5m]))
)

# 吞吐：generated token / 秒
sum by (model_name) (rate(vllm:generation_tokens_total[1m]))

# Block-level 命中率（5min 滑动）
sum by (model_name) (rate(vllm:prefix_cache_hits_total[5m]))
/
clamp_min(sum by (model_name) (rate(vllm:prefix_cache_queries_total[5m])), 1e-9)

# Token-level 节省比例
sum(rate(vllm:prompt_tokens_cached_total[5m]))
/
clamp_min(sum(rate(vllm:prompt_tokens_total[5m])), 1e-9)

# 实例间 QPS 偏差（max / min）
max by (model_name) (rate(vllm:request_success_total[1m]))
/
clamp_min(min by (model_name) (rate(vllm:request_success_total[1m])), 1e-9)

# 单实例 KV 利用率比集群中位数高 30%（routing 不均）
vllm:kv_cache_usage_perc
> on(model_name) (1.3 * quantile by(model_name)(0.5, vllm:kv_cache_usage_perc))

# 每 GPU 估算 TFLOPS
rate(vllm:estimated_flops_per_gpu_total[1m]) / 1e12

# 近似 MFU
rate(vllm:estimated_flops_per_gpu_total[1m]) / <peak_flops_per_gpu>

# 每 GPU 估算带宽 GB/s
(
  rate(vllm:estimated_read_bytes_per_gpu_total[1m])
  + rate(vllm:estimated_write_bytes_per_gpu_total[1m])
) / 1e9

# 近似 MBU
(
  rate(vllm:estimated_read_bytes_per_gpu_total[1m])
  + rate(vllm:estimated_write_bytes_per_gpu_total[1m])
) / <peak_bytes_per_second>

groups:
- name: vllm-slo
  interval: 30s
  rules:
  - alert: VLLMTTFTHigh
    expr: |
      histogram_quantile(0.99,
        sum by (model_name, le) (rate(vllm:time_to_first_token_seconds_bucket[5m]))
      ) > 1.0
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "TTFT p99 > 1s for {{ $labels.model_name }}"
      description: "Sustained 5 minutes. Check: queue depth, KV pressure, GPU sat."

  - alert: VLLMTPOTHigh
    expr: |
      histogram_quantile(0.99,
        sum by (model_name, le) (rate(vllm:request_time_per_output_token_seconds_bucket[5m]))
      ) > 0.1
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "TPOT p99 > 100ms for {{ $labels.model_name }}"

  - alert: VLLMAbortRateHigh
    expr: |
      sum by (model_name) (rate(vllm:request_success_total{finished_reason="abort"}[5m]))
      / sum by (model_name) (rate(vllm:request_success_total[5m]))
      > 0.01
    for: 10m
    labels:
      severity: critical
    annotations:
      summary: "Abort rate > 1% on {{ $labels.model_name }}"

- name: vllm-capacity
  interval: 30s
  rules:
  - alert: VLLMQueueDepthHigh
    expr: vllm:num_requests_waiting > 50
    for: 3m
    labels:
      severity: warning

  - alert: VLLMKVCacheNearFull
    expr: vllm:kv_cache_usage_perc > 0.9
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "KV cache > 90% on {{ $labels.instance }}"
      description: "Correlate with preemption, queue and SLO before mitigation."

  - alert: VLLMPreemptionsSpiking
    expr: rate(vllm:num_preemptions_total[5m]) > 0.5
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "Preempt rate > 0.5/s on {{ $labels.instance }}"

  - alert: VLLMIdleNoTraffic
    expr: |
      rate(vllm:request_success_total[10m]) == 0
      and vllm:engine_sleep_state == 0
    for: 15m
    labels:
      severity: info
    annotations:
      summary: "{{ $labels.instance }} has had no traffic in 15min but is awake"

- name: vllm-cache
  interval: 60s
  rules:
  - alert: VLLMPrefixCacheHitDrop
    expr: |
      (
        sum by (model_name) (rate(vllm:prefix_cache_hits_total[10m]))
        / sum by (model_name) (rate(vllm:prefix_cache_queries_total[10m]))
      ) < 0.3
    for: 30m
    labels:
      severity: info
    annotations:
      summary: "Prefix cache hit < 30% for 30min on {{ $labels.model_name }}"
      description: "Workload pattern shifted? Block size? Cache evicted?"

- name: vllm-failure
  rules:
  # 仅在 VLLM_COMPUTE_NANS_IN_LOGITS 启用时存在
  - alert: VLLMCorruptedRequests
    expr: increase(vllm:corrupted_requests_total[10m]) > 0
    labels:
      severity: critical
    annotations:
      summary: "Corrupted requests detected on {{ $labels.instance }}"
      description: "KV cache integrity failure. Check NCCL / disaggregated path."

  - alert: VLLMScrapeDown
    expr: up{job="vllm"} == 0
    for: 2m
    labels:
      severity: critical

1. "TTFT p99 是不是飙了？"        → SLO 行第 1 panel
2. "TPOT p99 是不是飙了？"        → SLO 行第 2 panel
3. "队列是不是积压了？"           → Capacity 行第 1 panel
4. "KV 满了吗？preempt 多吗？"    → Capacity 行第 2/3 panel
5. "scrape 还在吗？"             → grafana 顶部 + Prometheus targets

# 标准 Prometheus scrape config
- job_name: vllm
  scrape_interval: 15s
  static_configs:
    - targets: ['vllm-pod-1:8000', 'vllm-pod-2:8000', ...]
  # 或者用 kubernetes_sd_configs 自动发现

# docker-compose.yml（开发/测试用）
services:
  prometheus:
    image: prom/prometheus:<pinned-version-or-digest>
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./vllm-rules.yml:/etc/prometheus/vllm-rules.yml
    ports: ["9090:9090"]

  grafana:
    image: grafana/grafana:<pinned-version-or-digest>
    ports: ["3000:3000"]
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
    volumes:
      - ./grafana-dashboards:/var/lib/grafana/dashboards
      - ./grafana-provisioning:/etc/grafana/provisioning

  alertmanager:
    image: prom/alertmanager:<pinned-version-or-digest>
    ports: ["9093:9093"]

  vllm:
    image: vllm/vllm-openai:<validated-version-or-digest>
    command: --model facebook/opt-125m --port 8000
    ports: ["8000:8000"]

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 30s

rule_files:
  - vllm-rules.yml

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

scrape_configs:
  - job_name: vllm
    static_configs:
      - targets: ['vllm:8000']