diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.827b_bf16_mi300x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.827b_bf16_mi300x_mtp.sh new file mode 100755 index 0000000000..c98da1b09d --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/qwen3.827b_bf16_mi300x_mtp.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Qwen3.8-27B (native BF16 checkpoint of the dense hybrid +# attention model: 48 linear-attention and 16 full attention layers) on one +# MI300X, served by vLLM with the checkpoint's own MTP head (one MTP layer, +# mtp_num_hidden_layers=1) drafting three tokens per step. +# https://recipes.vllm.ai/Qwen/Qwen3.8-27B +# https://huggingface.co/Qwen/Qwen3.8-27B +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC ISL OSL RANDOM_RANGE_RATIO RESULT_FILENAME MAX_MODEL_LEN EVAL_ONLY RUN_EVAL THINKING_MODE + +if [[ -n "${SLURM_JOB_ID:-}" ]]; then + check_env_vars SLURMD_NODENAME + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ "$TP" -ne 1 ]]; then + echo "This recipe serves Qwen3.8-27B on a single GPU; got TP=$TP" >&2 + exit 1 +fi + +# The measured BF16 AL below belongs only to thinking-on, three-token MTP. +if [[ "$THINKING_MODE" != thinking_on ]]; then + echo "This recipe requires THINKING_MODE=thinking_on; got $THINKING_MODE" >&2 + exit 1 +fi + +# Keep the source FP8 recipe's three-token native MTP workload with BF16 weights. +NUM_SPEC_TOKENS=3 + +# gfx942: the ROCm vLLM arms run Triton attention and keep the KV cache in bf16 +# (no calibrated fp8 attention scales on this card); breakable piecewise graph +# capture is off as in the MiniMax-M3 and DSv4.1 Flash gfx942 arms. +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 +export PYTHONNOUSERSITE=1 + +rocm-smi --showmeminfo vram || true + +# Complete/resume partial downloads instead of trusting nonempty directories. +if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi + +SERVER_LOG=/workspace/server.log + +# Serve the matrix context (isl + osl + slack), not the checkpoint's 262K; +# accuracy evals use the eval context benchmark_lib derives. +MODEL_LEN="$MAX_MODEL_LEN" +if [[ "$EVAL_ONLY" == true ]]; then + setup_eval_context + MODEL_LEN="$EVAL_MAX_MODEL_LEN" +fi + +# vLLM's default max-num-seqs (1024) exceeds the GDN/Mamba cache blocks that fit +# next to the weights on the smaller cards (472 next to the bf16 weights on an +# 80 GB H100, run 35357364404) and engine start aborts before graph capture. +# Size the scheduler batch to the sweep point instead; the accuracy eval serves +# up to 256 concurrent requests. +MAX_NUM_SEQS=$(( CONC > 16 ? CONC : 16 )) +if [[ "$EVAL_ONLY" == true ]]; then + MAX_NUM_SEQS=256 +fi + +# Pyxis shares the host network; port 8888 can already belong to a host service. +select_available_server_port + +# Native MTP: no draft model, the head ships in the checkpoint (mtp.* tensors). +# BF16 thinking_on[3] from the measured SPEED-Bench curve in PR #3304. +# Any accuracy run keeps real verification, including combined throughput/eval. +if [[ "$EVAL_ONLY" == true || "$RUN_EVAL" == true ]]; then + SPEC_CONFIG=$(printf '{"method":"mtp","num_speculative_tokens":%d}' "$NUM_SPEC_TOKENS") +else + SPEC_CONFIG=$(printf '{"method":"mtp","num_speculative_tokens":%d,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' "$NUM_SPEC_TOKENS") +fi + +start_gpu_monitor + +VLLM_CMD=( + vllm serve "$MODEL" --served-model-name "$MODEL" + --host 0.0.0.0 --port "$PORT" + --tensor-parallel-size 1 + # Text-only serving: skip the vision tower of Qwen3_5ForConditionalGeneration. + --language-model-only + --trust-remote-code + # Target and native MTP head retain BF16 weights, compute and KV cache. + --dtype bfloat16 + --kv-cache-dtype auto + --attention-backend TRITON_ATTN + --max-model-len "$MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + # Every fixed-sequence request prefills its full prompt; no prefix-cache hits. + --no-enable-prefix-caching + --reasoning-parser qwen3 + --default-chat-template-kwargs '{"enable_thinking":true}' + --enable-auto-tool-choice --tool-call-parser qwen3_xml + --speculative-config "$SPEC_CONFIG" + --disable-uvicorn-access-log +) +printf '%q ' "${VLLM_CMD[@]}" | tee /workspace/vllm_command.txt +printf '\n' | tee -a /workspace/vllm_command.txt +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [[ "$EVAL_ONLY" == true ]]; then + run_eval --framework lm-eval --port "$PORT" + # Non-agentic evals must stage lm-eval's output into the workspace root. + append_lm_eval_summary +else + pip install -q datasets pandas + # The checkpoint chat template defaults to thinking on for client-side prompts. + run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --use-chat-template \ + --server-pid "$SERVER_PID" + if [[ "$RUN_EVAL" == true ]]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary + fi +fi + +stop_gpu_monitor diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9b4fcadeb7..1ebde95c52 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1472,3 +1472,22 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: search-space: # Follow upstream AMD Engram defaults; omit the CUDA-only config flag. - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + +qwen3.827b-bf16-mi300x-vllm-mtp: + image: vllm/vllm-openai-rocm:nightly-a8d1aa9c99b8698a2a78b611b7a10c30e6b3995b + model: Qwen/Qwen3.8-27B + model-prefix: qwen3.827b + runner: cluster:mi300x-amd + precision: bf16 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] } diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 4c6abfccfa..701fc99b07 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -222,6 +222,19 @@ Sources: [`AGENTS.md#non-negotiable-benchmark-invariants`](../AGENTS.md#non-nego 7. Add script + master entry + launcher routing + changelog together. 8. Run Bash syntax and generation checks. Inspect `spec-decoding`, draft/native method, token count, chat-template use, capture range, and resolved script. +### Qwen3.8-27B BF16 native MTP + +The MI300X recipe supports 1k/1k and 8k/1k at TP1, with concurrency 1–128. + +The BF16 vLLM recipes use thinking on with three native MTP draft tokens. Throughput +uses `rejection_sample_method: synthetic` and `synthetic_acceptance_length: 2.51`, +from the BF16 `thinking_on[3]` measurement in [#3304](https://github.com/SemiAnalysisAI/InferenceX/pull/3304). +The server explicitly enables thinking in its default chat-template kwargs; the +fixed-sequence client uses the checkpoint's thinking-on default via `--use-chat-template`. +The recipes require `THINKING_MODE=thinking_on` so another mode cannot reuse this AL. +Runs requesting accuracy through either `EVAL_ONLY` or `RUN_EVAL` use real MTP +verification. Target weights, native MTP weights and KV cache retain BF16. + ### DeepSeek-V4.1-Flash DSpark The GB200 DSpark recipe uses a minimum CUDA graph capture size of 64 tokens to cover concurrent AgentX subagents. This raises c1/c2/c4 from 8/16/32 to 64; c8 and above retain their existing sizes. The full trace, AL 3.51, and Engram UVA settings are preserved; low-concurrency tail latency improvements require CI confirmation. diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index f877035f18..6b00d29d55 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -199,6 +199,19 @@ llm-d 不是 srt-slurm 路径:InferenceX 自己持有 Slurm allocation,并 7. 同时添加脚本 + 主配置条目 + launcher 路由 + changelog。 8. 运行 Bash 语法和生成检查;检查 `spec-decoding`、draft/native 方法、token 数、chat-template 使用、capture 范围和解析出的脚本。 +### Qwen3.8-27B BF16 原生 MTP + +MI300X 配方支持 TP1 的 1k/1k 和 8k/1k,并发范围为 1–128。 + +BF16 vLLM 配方启用 thinking,使用 3 个原生 MTP 草稿 token。吞吐测试设置 +`rejection_sample_method: synthetic` 和 `synthetic_acceptance_length: 2.51`, +对应 [#3304](https://github.com/SemiAnalysisAI/InferenceX/pull/3304) 中 BF16 的 `thinking_on[3]` 测量值。 +服务端在默认 chat-template kwargs 中显式开启 thinking;固定序列长度客户端通过 +`--use-chat-template` 使用 checkpoint 默认开启的 thinking 模式。 +配方要求 `THINKING_MODE=thinking_on`,避免其他模式误用此 AL。 +通过 `EVAL_ONLY` 或 `RUN_EVAL` 请求准确率评测时,使用真实 MTP 验证。 +目标权重、原生 MTP 权重及 KV cache 均保持 BF16。 + ### DeepSeek-V4.1-Flash DSpark GB200 的 DSpark 配方将 CUDA graph 最小捕获范围设为 64 tokens,以覆盖 AgentX 子代理并发。这会将 c1/c2/c4 的上限从 8/16/32 提升至 64;c8 及以上保持原有大小。完整轨迹、AL 3.51 和 Engram UVA 配置保持不变;需通过 CI 验证低并发尾延迟改善。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5b4309f5f..b85b2619d6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8455,3 +8455,24 @@ description: - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274 + +- config-keys: + - qwen3.827b-bf16-mi300x-vllm-mtp + description: + - "Add MI300X Qwen3.8-27B BF16 vLLM TP1 native MTP with three draft tokens, BF16 KV cache, chat-templated 1k1k prompts, and concurrency 1-128; rerun the workload from #3294 with the unquantized checkpoint and the same pinned image" + - "新增 MI300X Qwen3.8-27B BF16 vLLM TP1 原生 MTP 配方:预测 3 个 token,使用 BF16 KV cache、经 chat template 处理的 1k1k 提示词及并发 1-128;使用原始未量化 checkpoint 和相同固定镜像重跑 #3294 的工作负载" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3307 + +- config-keys: + - qwen3.827b-bf16-mi300x-vllm-mtp + description: + - "Use thinking-on synthetic AL 2.51 with three native MTP draft tokens for MI300X Qwen3.8-27B BF16 throughput, matching the BF16 SPEED-Bench curve in #3304; accuracy runs retain real verification" + - "MI300X Qwen3.8-27B BF16 吞吐测试使用 thinking 开启、3 个原生 MTP 草稿 token 和合成 AL 2.51,对应 #3304 的 BF16 SPEED-Bench 曲线;准确率评测保留真实验证" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3307 + +- config-keys: + - qwen3.827b-bf16-mi300x-vllm-mtp + description: + - "Add 8k1k at concurrency 1-128 for MI300X Qwen3.8-27B BF16 vLLM TP1 native MTP, retaining thinking-on synthetic AL 2.51 with three draft tokens and the existing image" + - "为 MI300X Qwen3.8-27B BF16 vLLM TP1 原生 MTP 新增并发 1-128 的 8k1k 场景,保持 thinking 开启、合成 AL 2.51、3 个草稿 token 及原有镜像" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3307