vLLM 部署前沿大模型
从显存算术到并行拓扑

# bytes per token, per sequence
KV = 2 × n_layers × n_kv_heads × head_dim × dtype_bytes
#        ↑ K 和 V 各一份   ↑ 唯一的架构差异在这里

# MHA: n_kv_heads = n_heads        → 1×    
# GQA: n_kv_heads = n_heads / g    → 1/g   
# MQA: n_kv_heads = 1              → 1/n_heads

# example: Llama-3.3-70B, bf16 (64 q-heads / 8 kv-heads, g=8)
# 2 × 80 × 8 × 128 × 2 = 327,680 B ≈ 320 KiB / token
# 同一个模型若用 MHA（64 kv-heads）→ 2.56 MiB/token，整整 8 倍
# 8k 上下文单请求就要 2.6 GB —— 这就是 GQA 模型的并发天花板

# MLA 只缓存低秩潜向量 + RoPE 分量，不存完整 K/V
KV = n_layers × (kv_lora_rank + qk_rope_head_dim) × dtype_bytes

# example: DeepSeek-V3 / R1, bf16
# 61 × (512 + 64) × 2 = 70,272 B ≈ 68.6 KiB / token

# example: GLM-5, fp8 (1 byte)
# 78 × (512 + 64) × 1 = 44,928 B ≈ 43.9 KiB / token
# 同样 fp8 下，GLM-5 每 token 只有 Llama-70B 的 1/7

INFO [kv_cache_utils.py] GPU KV cache size: 643,232 tokens
INFO [kv_cache_utils.py] Maximum concurrency for 40,960 tokens per request: 15.70x

vllm serve zai-org/GLM-5-FP8 \
  --served-model-name glm-5 \
  --tokenizer-mode glm5_next \
  --enable-auto-tool-choice \
  --tool-call-parser glm47 \
  --reasoning-parser glm47 \
  --chat-template-content-format=string

# MTP：使用 checkpoint 内置的多 token 预测头
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'

# DSpark：DeepSeek-V4-0813 融合 checkpoint，K 为草稿长度
--speculative-config '{"method":"dspark","num_speculative_tokens":6,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'

# EAGLE3：需要显式指定 draft 模型
--speculative-config '{"method":"eagle3","model":"/models/eagle3-draft","num_speculative_tokens":3}'

# n-gram：无额外模型，适合重复性强的文本
--speculative-config '{"method":"ngram","num_speculative_tokens":5,"prompt_lookup_max":4}'

# A. 纯 TP8 —— 所有层都切，每层两次 all-reduce
vllm serve deepseek-ai/DeepSeek-V4-Pro \
  --tensor-parallel-size 8

# B. DP8 + EP —— 注意力复制、专家分片，只有 MoE 层 all-to-all
vllm serve deepseek-ai/DeepSeek-V4-Pro \
  --data-parallel-size 8 \
  --enable-expert-parallel \
  --all2all-backend deepep_low_latency

# 单卡（A100 80G / H100 / H200 / B200 均可）
vllm serve openai/gpt-oss-120b

# TP4（多卡分摊，降低单卡显存压力）
vllm serve openai/gpt-oss-120b --tensor-parallel-size 4

# 启用工具调用（OpenAI harmony 格式）
vllm serve openai/gpt-oss-120b \
  --tool-call-parser openai \
  --enable-auto-tool-choice

# Blackwell 上 FlashInfer TRTLLM MXFP4 MoE 会自动选中；
# 如需显式固定：
vllm serve openai/gpt-oss-120b \
  --moe-backend flashinfer_trtllm \
  --quantization-config.moe.activation mxfp8

# Official OpenAI SDK works unchanged against vLLM
from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY")
r = client.chat.completions.create(
    model="openai/gpt-oss-120b",
    messages=[
        {"role": "system", "content": "Reasoning: high"},
        {"role": "user", "content": "Explain why eigenvalues matter."},
    ],
)
print(r.choices[0].message.content)

# 8×H200：DP8 + EP，跳过视觉编码器，把显存全给 KV
vllm serve Qwen/Qwen3.5-397B-A17B-FP8 \
  -dp 8 \
  --enable-expert-parallel \
  --language-model-only \
  --reasoning-parser qwen3 \
  --enable-prefix-caching

vllm serve Qwen/Qwen3.5-397B-A17B-FP8 \
  -dp 8 \
  --enable-expert-parallel \
  --mm-encoder-tp-mode data \
  --mm-processor-cache-type shm \
  --reasoning-parser qwen3 \
  --enable-prefix-caching

# 加工具调用
# --enable-auto-tool-choice --tool-call-parser qwen3_coder

# 注意：MTP 会占用 KV，高并发下反而降吞吐
vllm serve Qwen/Qwen3.5-397B-A17B-FP8 \
  --tensor-parallel-size 8 \
  --speculative-config '{"method":"mtp","num_speculative_tokens":1}' \
  --reasoning-parser qwen3

# H200 上 1M 上下文放不下全部 KV，官方建议封顶 800K
vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \
  --data-parallel-size 8 \
  --enable-expert-parallel \
  --max-model-len 800000 \
  --kv-cache-dtype fp8 \
  --block-size 256 \
  --tokenizer-mode deepseek_v4 \
  --reasoning-parser deepseek_v4 \
  --tool-call-parser deepseek_v4 \
  --enable-auto-tool-choice \
  --trust-remote-code

# 每个 4-GPU tray 起本地 4 个 DP rank，两 tray 合成 DP8
export VLLM_USE_V2_MODEL_RUNNER=1
export VLLM_DSV4_MEGA_FP8_COMBINE=1

vllm serve /models/DeepSeek-V4-Pro-0813 \
  --data-parallel-size 8 \
  --data-parallel-size-local 4 \
  --data-parallel-start-rank $((4 * NODE_RANK)) \
  --data-parallel-rpc-port 13345 \
  --data-parallel-hybrid-lb \
  --enable-expert-parallel \
  --enable-ep-weight-filter \
  --enable-cumem-allocator \
  --moe-backend deep_gemm_amxf4_mega_moe \
  --numa-bind --numa-bind-nodes 0 0 1 1 \
  --tokenizer-mode deepseek_v4 \
  --trust-remote-code \
  --max-model-len 1048576 \
  --kv-cache-dtype fp8 \
  --block-size 256 \
  --no-disable-hybrid-kv-cache-manager \
  --disable-flashinfer-autotune \
  --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}' \
  --speculative-config '{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block"}'

# 2 个 GB300 tray 或 1 台 B300 节点，TP8 基础布局
docker run --gpus all --ipc=host --shm-size 32g \
  -p 8000:8000 \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  vllm/vllm-openai:kimi-k31 \
  moonshotai/Kimi-K3.1-Pro \
  --tensor-parallel-size 8 \
  --trust-remote-code \
  --max-model-len 1048576 \
  --gpu-memory-utilization 0.85 \
  --enable-prefix-caching \
  --tool-call-parser kimi_k3 \
  --reasoning-parser kimi_k3 \
  --enable-auto-tool-choice

# Decode Context Parallelism：MLA 潜变量与索引器 KV 分片，
# KV 容量约为 TP8 的 7 倍（约 18.5M vs 2.6M token），代价是 decode 慢 7–25%
vllm serve moonshotai/Kimi-K3 \
  --tensor-parallel-size 8 \
  --decode-context-parallel-size 8 \
  --dcp-comm-backend a2a \
  --attention-backend TOKENSPEED_MLA \
  --kv-cache-dtype fp8 \
  --mamba-ssm-cache-dtype bfloat16 \
  --enable-prefix-caching --prefix-match-unit 128 \
  --max-model-len 1048576 \
  --gpu-memory-utilization 0.88 \
  --speculative-config '{"model":"Inferact/Kimi-K3-DSpark","method":"dspark","num_speculative_tokens":6,"attention_backend":"TOKENSPEED_MLA","draft_sample_method":"probabilistic"}' \
  --tool-call-parser kimi_k3 --reasoning-parser kimi_k3 \
  --language-model-only

vllm serve Qwen/Qwen3.8-Flash-Next-FP8 \
  --tensor-parallel-size 4 \
  --enable-expert-parallel \
  --gpu-memory-utilization 0.90 \
  --max-num-seqs 256 \
  --enable-prefix-caching \
  --kv-cache-dtype fp8 \
  --attention-config.indexer_kv_dtype fp8 \
  --speculative-config '{"method":"mtp","num_speculative_tokens":3}' \
  --enable-auto-tool-choice \
  --tool-call-parser qwen3_coder \
  --reasoning-parser qwen3

# Hopper 上必须用 Triton MoE 后端
vllm serve Qwen/Qwen3.8-Flash-Next-FP8 \
  --tensor-parallel-size 8 \
  --enable-expert-parallel \
  --moe-backend triton \
  --gpu-memory-utilization 0.85 \
  --max-num-seqs 256 \
  --enable-prefix-caching \
  --kv-cache-dtype fp8 \
  --attention-config.indexer_kv_dtype fp8 \
  --speculative-config '{"method":"mtp","num_speculative_tokens":3}' \
  --enable-auto-tool-choice \
  --tool-call-parser qwen3_coder \
  --reasoning-parser qwen3

vllm serve zai-org/GLM-5-FP8 \
  --tensor-parallel-size 8 \
  --speculative-config.method mtp \
  --speculative-config.num_speculative_tokens 3 \
  --tool-call-parser glm47 \
  --reasoning-parser glm47 \
  --enable-auto-tool-choice \
  --chat-template-content-format=string \
  --served-model-name glm-5-fp8

# 1) 起一个带工具调用的服务
vllm serve openai/gpt-oss-120b \
  --served-model-name my-model \
  --enable-auto-tool-choice \
  --tool-call-parser openai

# 2) 指向它启动 Claude Code
ANTHROPIC_BASE_URL=http://localhost:8000 \
ANTHROPIC_API_KEY=dummy \
ANTHROPIC_AUTH_TOKEN=dummy \
ANTHROPIC_DEFAULT_OPUS_MODEL=my-model \
ANTHROPIC_DEFAULT_SONNET_MODEL=my-model \
ANTHROPIC_DEFAULT_HAIKU_MODEL=my-model \
claude

vllm bench serve \
  --model zai-org/GLM-5-FP8 \
  --dataset-name random \
  --random-input 8000 \
  --random-output 1024 \
  --request-rate 10 \
  --num-prompts 32 \
  --ignore-eos

docker run --runtime nvidia --gpus all \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  -e HF_TOKEN=$HF_TOKEN \
  -p 8000:8000 \
  --ipc=host \
  vllm/vllm-openai:latest \
  --model Qwen/Qwen3-0.6B

# 老驱动（数据中心卡）可开启 CUDA 兼容层
-e VLLM_ENABLE_CUDA_COMPATIBILITY=1

curl -s http://localhost:8000/metrics | grep -E 'kv_cache_usage|num_requests_waiting|num_preemptions|prefix_cache'