From b0dae8b3309cfa0e7f2322e3a2bdb038e68cc5e4 Mon Sep 17 00:00:00 2001 From: vllm-omni-review-bot Date: Fri, 11 Sep 2026 15:21:51 +0800 Subject: [PATCH] knowledge: daily intake batch 2026-09-11 --- .../vllm-omni/components/diffusion/rules-attention.md | 9 ++++++++- .../vllm-omni/components/serving/rules-request-input.md | 9 ++++++++- .../vllm-omni/components/serving/rules-stage-startup.md | 9 ++++++++- knowledge/repos/vllm-omni/models/cosyvoice3/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/higgs-audio/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/ltx2/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/omnivoice/rules.md | 9 ++++++++- knowledge/repos/vllm-omni/models/pi0/rules.md | 9 ++++++++- 8 files changed, 64 insertions(+), 8 deletions(-) diff --git a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md index 7ebc0620a..5aeb99bf3 100644 --- a/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md +++ b/knowledge/repos/vllm-omni/components/diffusion/rules-attention.md @@ -1,7 +1,7 @@ --- title: "Diffusion attention 规则" created: 2026-09-02 -updated: 2026-09-10 +updated: 2026-09-11 type: rule tags: [vllm-omni, components, diffusion] sources: ["PR #5543", "PR #5866", "PR #5887", "PR #5891", "PR #5897", "PR #5997", "PR #6000", "PR #6037", "PR #6518", "PR #6563", "PR #6724", "PR #6909", docs/design/feature/skip_softmax.md, docs/user_guide/diffusion/attention_backends.md, docs/user_guide/diffusion/attention_backends/trtllm.md, docs/user_guide/diffusion/attention_backends/fastvideo_vsa.md, docs/user_guide/diffusion/attention_backends/rainfusion.md, vllm_omni/config/omni_config.py, vllm_omni/config/stage_config.py, vllm_omni/diffusion/attention/backends/abstract.py, vllm_omni/diffusion/attention/backends/fastvideo_vsa.py, vllm_omni/diffusion/attention/backends/flash_attn.py, vllm_omni/diffusion/attention/backends/rainfusion_attn.py, vllm_omni/diffusion/attention/parallel/ulysses.py, vllm_omni/diffusion/diffusion_kv/paged_attention_adapter.py, vllm_omni/diffusion/models/minimax_h3/denoise_loop.py, vllm_omni/diffusion/models/minimax_h3/packed_sequence.py, vllm_omni/diffusion/data.py, vllm_omni/engine/arg_utils.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/cli/serve.py, vllm_omni/platforms/cuda/platform.py, vllm_omni/platforms/npu/platform.py, tests/config/test_omni_config.py, tests/diffusion/attention/test_fastvideo_vsa.py, tests/diffusion/attention/test_flash_attn.py, tests/diffusion/attention/test_attention_config.py, tests/diffusion/attention/test_piecewise_attn.py, tests/diffusion/attention/test_rainfusion_plan.py, tests/diffusion/attention/test_ulysses_uaa.py, tests/diffusion/diffusion_kv/test_paged_attention_adapter.py, tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py, tests/diffusion/cache/test_teacache_extractors.py, "PR #5500", "vllm_omni/diffusion/models/ltx2/ltx2_transformer.py", "PR #6070", "vllm_omni/diffusion/attention/backends/cudnn_attn.py", "PR #5614", "PR #5194", "vllm_omni/diffusion/models/hidream_o1_image/hidream_o1_image_transformer.py", "vllm_omni/diffusion/models/hidream_o1_image/pipeline_hidream_o1_image.py", "PR #6181", "vllm_omni/diffusion/cache/teacache/extractors.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py", "vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py", "PR #5717", "PR #6871"] @@ -217,3 +217,10 @@ confidence: high - 强制：仅当 `seq_lengths` 存在真实 padding（至少一档短于 `max(seq_lengths)`）时才物化 `(batch, max_len)` bool mask；单样本或全员等长 batch 必须返回 `None`，让 backend 直接走 dense path。all-true CUDA mask 会触发 `torch.any(~mask)` 的 device→host sync，不能当作“无害 no-op”。 - 禁止：无条件 `new_zeros` + 填 True；把 all-true device mask 交给 FlashAttention 只为“形状完整”；把省略 mask 说成改变了可变长 batch 的有效 token 集合。 - 验收：等长与单样本断言 helper/`forward` 得到 `None`；可变长断言 mask shape 与 True/False 边界；不得用一次端到端加速数字代替 sync 合同。^[PR #6871] + +## DIFF-1ai — 显式 Hub FlashAttention 必须在选择时解析构建并失败可见 + +- 触发：修改 `FLASH_ATTN_HUB` / `FLASH_ATTN_3_HUB` 选择、`flash_attn_hub._load_hub_module` / `_get_hub_module`、`kernels` pin，或依赖 Hub FA 的 accuracy/serving 路径。 +- 强制：显式 Hub backend 在 platform 返回 path 前必须走与执行相同的 loader：对 `kernels-community/flash-attn2` 或 `flash-attn3` 依次尝试 `get_kernel(..., version=1)` 再 `version=2`；任一命中即缓存。variant miss 或加载失败必须 `RuntimeError`，说明无兼容 torch/CUDA build，并提示 `kernels>=0.16.1`（含 stable-ABI 匹配）或改选其他 backend。`requirements` 与 LTX/NATTEN remediation 文案中的 `kernels` / `transformers` floor 必须与该 loader 所需 companion pin 同步（当前为 `kernels==0.16.1` 与 `transformers>=5.13.0,<5.15`）。 +- 禁止：显式 Hub 选择在 miss 后静默落到 native/SDPA；preflight 只试 `version=1` 或再试无 version 的第三态；把 accuracy 探针在 Hub 不可用时双方改跑 SDPA 仍当作 FA3-vs-FA3 门禁；把 import/`has_flash_attn` 成功误当成 Hub build 已解析。 +- 验收：CPU 覆盖 v1 miss→v2 hit 的版本序列、双 Hub backend 的 variant miss 均 raise、显式选择无 silent fallback；真实 torch/CUDA 环境再证明 stable-ABI 或 exact-tag build 可加载。不得用 SDPA 噪声 SSIM 解释 Hub miss。^[PR #7185] diff --git a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md index 495c86590..546d73a42 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-request-input.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-request-input.md @@ -1,7 +1,7 @@ --- title: "请求输入合同" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-11 type: rule tags: [vllm-omni, components, serving] sources: ["PR #3805", "PR #5374", "PR #5885", "PR #6598", vllm_omni/data_entry_keys.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/entrypoints/openai/, vllm_omni/entrypoints/omni_base.py, vllm_omni/engine/orchestrator.py, vllm_omni/inputs/, tests/engine/test_async_omni_engine_input.py, tests/engine/test_orchestrator_error_handling.py, tests/entrypoints/test_omni_entrypoints.py, tests/entrypoints/openai_api/test_invalid_audio_speech.py, tests/entrypoints/openai_api/test_serving_speech.py, "PR #5181", "PR #6182"] @@ -139,3 +139,10 @@ engine 生命周期见 [engine 生命周期规则](rules-engine-lifecycle.md)； - 强制：transform 只操作 Stage-0 copy；原始 prompt 保留给 downstream，并在 transform 前后保持同一 global request ID。只把已处理的 metadata 合并回原始视图，先移除 transform-owned stale keys。临时目录只通过内部 `REQUEST_ARTIFACT_DIRS_KEY` 交给 request state；preprocess、companion build、enqueue 前失败立即回收，admit 后由 orchestrator 在所有 terminal cleanup 路径回收，且内部 key 不得进入 stage payload。 - 禁止：把 transformed prompt 当作 downstream 原始媒体、让旧 prepared descriptor 跨请求复用、在 ownership 已交给 orchestrator 后由 frontend 提前删除，或让异常路径泄漏转码目录。 - 验收：覆盖 transform replacement/copy、request ID 与 metadata merge、成功 terminal/abort、preprocess/companion/enqueue 异常以及无 artifact control；断言 downstream 看见原始媒体+允许的 processed meta，目录恰好由当前 owner 回收，内部 key 未传输。^[PR #5885] + +## SERV-4r — video 参考图必须在 RGB 转换前执行像素上限并映射解码器炸弹 + +- 触发：修改 `/v1/videos` 的 `image_reference` / `input_reference(s)`、MiniMax H3 multipart 图片上传，或 `video_api_utils._decode_image_bytes` / `_validate_image_pixel_limit`。 +- 强制：`Image.open` 后、`convert("RGB")` 前按 header 尺寸检查 `width * height`；当 `VLLM_MAX_IMAGE_PIXELS > 0` 且超出时拒绝。`=0` 禁用检查，恰等于上限放行。Pillow `DecompressionBombError` 必须映射为专用像素上限 client error（HTTP 400），不得落入泛化 “not a valid image” 后再当 video 重试。H3 multipart 与共享 decode helper 共用同一合同。 +- 禁止：先全量 decode/convert 再检像素；把 decoder bomb 误判为可 video-fallback 的无效图；用 500 或静默接受放大资源占用。 +- 验收：覆盖超限在 convert 前失败、disabled/inclusive 边界、bomb→专用 400、以及 H3 multipart 字段拒绝。^[PR #6963] diff --git a/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md b/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md index 2e5eb0086..f9352cd6c 100644 --- a/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md +++ b/knowledge/repos/vllm-omni/components/serving/rules-stage-startup.md @@ -1,7 +1,7 @@ --- title: "Stage 启动与设备布局规则" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-11 type: rule tags: [vllm-omni, components, serving] sources: ["PR #6050", "PR #5445", "Issue #5003", "PR #5742", vllm_omni/engine/stage_init_utils.py, tests/engine/test_stage_device_layout.py] @@ -33,3 +33,10 @@ confidence: high - 强制：对声明 `runtime.devices` 的每个 LLM stage，在 `create_engine_config` 或 worker/executor 创建前，以同一公式验证并拆分设备：每个本地 replica 需要 `tensor_parallel_size × data_parallel_size_local × pipeline_parallel_size`；`data_parallel_size_local` 未设置时才回退到全局 DP，值为 `0` 的 head process 不做本地设备验证。显式列表可为单 replica template（恰为每副本大小）或完整 pool（该大小 × `num_replicas`）；未声明 `devices` 时保留 launcher 分配语义。仅当去掉 TP 后设备数恰好有效，错误才说明 top-level TP 会广播到全部 stage，并要求在每个 stage override 中一并设置 TP 与 `devices`；其他 TP/DP/PP/replica 不匹配给通用维度说明。 - 禁止：按集群全局 DP 校验本地 `runtime.devices`；让 guard 与 replica splitter 使用不同的每副本宽度；在 engine config/worker 启动后才暴露 `local rank ... out of bounds`；把 PP 或其他布局错误一律归因于 top-level TP；因为没有显式 devices 而拒绝 vLLM 的设备分配。 - 验收：覆盖 TP 广播的单卡 stage 在 engine-config/executor 前失败、匹配 TP/device 通过、未声明 devices 跳过、local-DP 小于 global-DP 与 local-DP=0、PP/local-DP 加多副本的 template 和 full-pool 拆分，以及 PP-only mismatch 不含 TP 专属 workaround。^[Issue #5003] ^[PR #5742] + +## SERV-12c — 串行 LLM 副本初始化必须按物理 GPU 重叠连通分量分组 + +- 触发：修改 `stage_runtime` 的 replica init group、`parse_physical_device_ids`、`device_overlap_group_keys`、`acquire_device_locks` / `spawn_device_lock`，或多 stage 共享/重叠 `runtime.devices` 的启动路径。 +- 强制：本地串行 LLM replica 按已解析物理 device **集合**的连通分量分组：集合相交（含 `{0,1}` 与 `{0}`、`"1,0"` 与 `"0,1"`、传递链）共用同一 `device-group:` 并顺序初始化；不相交集合才并行。非整数 visibility（UUID/MIG）或 unresolved 解析为 `None` 时，该批串行 LLM 收拢为单一 `device-group:*`。diffusion / remote / `parallel_stage_init` 保持既有 per-replica override key。 +- 禁止：仅用精确 device **字符串**分组，使重叠但不等的集合进入并行线程后，同进程对已持有 inode 的 `flock(LOCK_EX|LOCK_NB)` 自相争并在仍持有 `spawn_device_lock` 时空转到 `stage_init_timeout`。 +- 验收：覆盖重叠/传递重叠同组、不相交异组、非整数 devices 收拢；多 stage `0,1`+`0` 启动不得因 device-lock timeout 无锁继续。^[PR #7328] diff --git a/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md b/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md index 985107088..feba3aca8 100644 --- a/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md +++ b/knowledge/repos/vllm-omni/models/cosyvoice3/rules.md @@ -1,7 +1,7 @@ --- title: "CosyVoice3 规则" created: 2026-09-04 -updated: 2026-09-05 +updated: 2026-09-11 type: rule tags: [vllm-omni, models] sources: ["PR #5673", "PR #6955", vllm_omni/model_executor/models/cosyvoice3/code2wav_core/cfm.py, vllm_omni/model_executor/models/cosyvoice3/flow_estimator_trt.py, tests/model_executor/models/cosyvoice3/test_cosyvoice3_components.py, benchmarks/tts/benchmark_cosyvoice3_trt_streams.py] @@ -50,3 +50,10 @@ confidence: high 证明 replace 失败保留旧 plan、cleanup 失败保留 replace error、partial write 清理 owned tmp、 collision 保留 foreign tmp，以及两条同步 publisher 使用不同 source path 且仅留下完整 final plan。 ^[PR #6955] + +## COSYVOICE3-2a — deploy 必须在 name 推断失败时显式绑定 `pipeline: cosyvoice3` + +- 触发：修改 `vllm_omni/deploy/cosyvoice3.yaml`、CosyVoice3 的 `StageConfigFactory` / deploy 解析，或依赖 HF cache `models--…/snapshots/` 且 root `config.json` 无识别元数据的启动路径。 +- 强制：shipping deploy 必须声明 `pipeline: cosyvoice3`（或等价显式 pipeline key），使 hash-named snapshot 在 `try_infer_model_type` / name-fallback 失败时仍解析为已注册的两 LLM stage（talker + code2wav），而不是 default diffusion stage。 +- 禁止：仅依赖 basename/hash 推断；让空 `config.json` snapshot 落入 Diffusers `model_index.json` 查找并在启动时报 `Diffusers pipeline index not found`。 +- 验收：以空 `config.json` 的 `snapshots/<40-hex>` 布局回归 `create_from_model` 与 legacy stage 路径，断言 `pipeline_config.model_type == "cosyvoice3"`、两 stage 均为 LLM（非 DIFFUSION）且 model 路径保留。^[PR #6896] diff --git a/knowledge/repos/vllm-omni/models/higgs-audio/rules.md b/knowledge/repos/vllm-omni/models/higgs-audio/rules.md index 38ba1001d..8ea47c3a8 100644 --- a/knowledge/repos/vllm-omni/models/higgs-audio/rules.md +++ b/knowledge/repos/vllm-omni/models/higgs-audio/rules.md @@ -1,7 +1,7 @@ --- title: "Higgs-Audio 规则" created: 2026-09-05 -updated: 2026-09-05 +updated: 2026-09-11 type: rule tags: [vllm-omni, models, model-executor] sources: ["PR #6422", "PR #7065", vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py, vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_tokenizer.py, vllm_omni/worker/gpu_model_runner.py, tests/entrypoints/openai_api/test_serving_speech.py, tests/model_executor/models/higgs_audio_v3/test_higgs_audio_v3.py, tests/e2e/online_serving/test_higgs_audio_v3.py] @@ -54,3 +54,10 @@ confidence: high 一部分的 chunked prefill 和 legacy `-100` caller 注入正确 code rows；serving smoke 证明相同 路径重写时 cache salt 仍变化。H100 voice-clone E2E 未在该 PR 的 lanes 运行，不能把 unit coverage 宣称为 end-to-end 证据。^[PR #7065] + +## HIGGS-3a — 共享 reference-code encode 必须屏蔽请求取消并以 task 身份退休 + +- 触发：修改 Higgs-Audio V3 `_resolve_higgs_audio_v3_ref_codes`、artifact-keyed inflight map，或其他 TTS adapter 的 single-flight reference encode。 +- 强制：同一 `artifact_key` 的 cold encode 只创建一个共享 `asyncio.Task`；creator 与 waiter 都必须 `await asyncio.shield(task)`，请求取消不得取消共享 encode。inflight 条目只能在 task `done_callback` 里用 identity guard 弹出；未取消任务须 `t.exception()` 消费失败以免 “exception never retrieved”。成功路径仍 clone 后返回，cache 写入保持既有合同。 +- 禁止：直接 `await task` 让 caller cancellation 传入共享 flight；在 creator 的 `finally` 里按请求退出撤销 inflight（会让仍在跑的 task 失位并可能重复 encode）。 +- 验收：确定性 CPU 回归覆盖 creator+waiter 取消后幸存者仍从同一 encode 完成、全取消后失败路径无 loop exception handler 且 inflight 清空；与 MOSS single-flight 生命周期对齐但不改 cache key/容量/API。^[PR #7076] diff --git a/knowledge/repos/vllm-omni/models/ltx2/rules.md b/knowledge/repos/vllm-omni/models/ltx2/rules.md index 6f2f149b4..3316342c1 100644 --- a/knowledge/repos/vllm-omni/models/ltx2/rules.md +++ b/knowledge/repos/vllm-omni/models/ltx2/rules.md @@ -1,7 +1,7 @@ --- title: "LTX-2.5 DiffVAE 规则" created: 2026-09-04 -updated: 2026-09-05 +updated: 2026-09-11 type: rule tags: [vllm-omni, models, ltx2, diffusion] sources: ["PR #6189", "PR #7000", "PR #7020", recipes/LTX/LTX-2.5.md, requirements/common.txt, docs/user_guide/diffusion/attention_backends/huggingface_hub.md, vllm_omni/diffusion/models/ltx2/ltx2_components.py, vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder.py, vllm_omni/diffusion/models/ltx2/ltx2_diffusion_decoder_distributed.py, vllm_omni/diffusion/models/ltx2/ltx2_latents.py, vllm_omni/diffusion/models/ltx2/ltx2_request.py, vllm_omni/diffusion/models/ltx2/ltx2_runtime.py, tests/diffusion/models/ltx2/test_ltx25_pipeline.py, tests/diffusion/models/ltx2/test_ltx2_output_cuda.py, tests/diffusion/models/ltx2/test_ltx2_pipeline.py, tests/diffusion/models/ltx2/test_ltx2_vae.py] @@ -71,3 +71,10 @@ confidence: high 对比。有效 `[0, 1]` 的 `do_normalize=False` 输入须与 unclipped legacy path 比较；另测 intentional out-of-range defensive clamp。FP32 presentation bytes 必须 exact，BF16/FP16 相对 legacy FP32 presentation 最多相差一个 uint8 level；官方 similarity 仅是本 PR 所报 LTX matrix 的限定证据。^[PR #7000] + +## LTX-4 — LTX vocoder 的 CUDA 路径必须 scoped 启用并恢复 `cudnn.deterministic` + +- 触发：修改 `ltx2_runtime._run_ltx_vocoder` / `_deterministic_ltx_vocoder`、BWE vocoder 调用，或 accuracy reference runner 的官方 vocoder forward。 +- 强制：仅在 `device_type == "cuda"` 时进入 context：将 `torch.backends.cudnn.deterministic` 设为 `True`，成功与异常退出都恢复调用前值；不得改动其他 cuDNN 开关。保留既有 BWE FP32 autocast / MPS float 合同。官方 reference runner 对 pinned BWE vocoder 施加同一 scoped pin，使两侧音频可稳定对照。 +- 禁止：把 process-wide deterministic 永久打开；假设并发 worker 可安全争用该全局 flag；把偶发相对-L2 过线当作已消除算法漂移。 +- 验收：CPU 覆盖初始 True/False 与注入异常后均恢复，且 `enabled`/`benchmark`/`allow_tf32` 等其他设置不变；相似度/固定 mel 实验只作有界证据，须在相同硬件与序列化执行假设下解读。^[PR #7231] diff --git a/knowledge/repos/vllm-omni/models/omnivoice/rules.md b/knowledge/repos/vllm-omni/models/omnivoice/rules.md index f2744f2bf..7196927cc 100644 --- a/knowledge/repos/vllm-omni/models/omnivoice/rules.md +++ b/knowledge/repos/vllm-omni/models/omnivoice/rules.md @@ -1,7 +1,7 @@ --- title: "OmniVoice 规则" created: 2026-09-05 -updated: 2026-09-05 +updated: 2026-09-11 type: rule tags: [vllm-omni, models, model-executor, attention-runtime, ffn-runtime] sources: ["PR #6317", vllm_omni/model_executor/models/omnivoice/omnivoice_generator.py, vllm_omni/model_executor/models/omnivoice/fused_qkv_rope.py, tests/model_executor/models/omnivoice/test_cuda_graph_generator.py, tests/model_executor/models/omnivoice/test_fused_projection_load.py, tests/model_executor/models/omnivoice/test_fused_qkv_rope.py, tests/model_executor/models/omnivoice/test_mask_dtype.py, tests/model_executor/models/omnivoice/test_triton_kernels.py] @@ -30,3 +30,10 @@ confidence: high - 强制：checkpoint pack 顺序固定为 `qkv=[q,k,v]`、`gate_up=[gate,up]`；一旦某层开始 pack，缺少或仅出现部分 source shards 必须拒绝，不能让 fused parameter 保持随机初始化；未进入任何 fused-source group 的 shard 保持 generic loader 处理。attention prologue 只以 fp32 进行 Q/K RMSNorm、Q/K 权重计算与 Q/K RoPE，V 保持原 projection 值；随后按 GQA repeat K/V，并产出 contiguous `[batch, heads, sequence, head_dim]`（BHSD）。Triton 仅在 Triton 可用、CUDA tensor、QKV contiguous、head dim 为正的 2 的幂、Q 与 KV head 数都可被 8 整除且 GQA 可整除时启用；其他情况走 eager reference。residual 的加法/归一化代数不变，SwiGLU 保持 `silu(gate) * up` 的顺序。 - 禁止：改变 source shard packing 次序、接受 partial/missing pack group，或把没有 fused source 的整组 shards 误判为 partial；对 V 应用 Q/K norm 或 RoPE；重排 GQA 的 K/V 映射、返回非 contiguous BHSD；把 Triton 的存在或 CUDA 单项条件当作充分资格，或删除 eager fallback；为方便而给 generator mask 重新引入 fp32 默认。不得把本 PR 的 A800、CPU 或 CUDA 测试/性能证据外推为其他硬件、shape、后端或普遍 serving support。 - 验收：CPU 覆盖 pack order、全缺 source group 不触发 pack、任一 partial/missing 或错误 shape 拒绝，以及 eager 输出；CUDA+Triton 仅在上述 geometry 下对 eager reference 覆盖 fp32/fp16/bf16、Q/K norm+RoPE、未经变换的 V、正确 GQA index 与 contiguous BHSD；无 Triton、CPU、non-contiguous 或不合资格 geometry 必须回退。mask 测试覆盖 `model_dtype` 的 bool True→`0.0`、False→`-inf`、float 原样保留和 graph replay；graph/eager 及 residual/SwiGLU 回归保持既有精度合同。A800 上的 float16/float32 性能和 CPU/CUDA 单测只构成该 PR 的有界证据，不构成广泛性能或支持承诺。 ^[PR #6317] + +## OMNIVOICE-2a — OmniVoice 请求批必须使用 packed varlen CFG 布局并按请求解码 + +- 触发：修改 OmniVoice pipeline/generator 的 request batch、step execution、CFG 输入布局、`cu_seqs`、CUDA Graph 键，或 codec decoder 批处理。 +- 强制：声明 `supports_request_batch` 与 `supports_step_execution` 时，Generator 使用 token-major packed 布局 `[cond0, uncond0, cond1, uncond1, …]`，以 `cu_seqs` 表达 `2B` 条独立序列的累积边界，序列内双向、序列间互不可见；position id 按 `cu_seqs` 每段从 0 重置。CUDA Graph 键为 `(request_batch_size, token_bucket)`，静态输入按 token bucket 与 `2B+2` `cu_seqs` 捕获。每请求保留独立 RNG，避免 Gumbel 流交叉。完成请求用**精确长度** codec token **逐条** decode；不得为迁就 Decoder 批处理而把已 exact-length 的 token 再 pad 到 longest。 +- 禁止：把 cond/uncond 或跨请求 pad 成 `[batch, codebook, max_seq]` 当作默认热路径；共享 batch 级 RNG；在未证明 pad-Decoder 更快前恢复 padded Decoder batching。不得把注意力/layout 变更后的输出宣称为与旧 padded 路径 bit-exact。 +- 验收：覆盖 packed `cu_seqs` 边界、per-request seed 隔离、graph key/bucket、step lifecycle，以及 exact-length 单请求 decode；WER/parity 报告须标明 layout 变更。deploy 批参数（如 `max_num_seqs` / `request_batch_max_wait_ms`）只证明该配置，不构成通用吞吐保证。^[PR #6408] diff --git a/knowledge/repos/vllm-omni/models/pi0/rules.md b/knowledge/repos/vllm-omni/models/pi0/rules.md index 3d65e8b94..5cef69263 100644 --- a/knowledge/repos/vllm-omni/models/pi0/rules.md +++ b/knowledge/repos/vllm-omni/models/pi0/rules.md @@ -1,7 +1,7 @@ --- title: "Pi0 模型硬门禁" created: 2026-09-04 -updated: 2026-09-04 +updated: 2026-09-11 type: rule tags: [vllm-omni, models, diffusion, serving] sources: ["PR #4222", "vllm_omni/diffusion/models/pi0/config.py", "vllm_omni/diffusion/models/pi0/modeling_pi0.py", "vllm_omni/diffusion/models/pi0/pipeline_pi0.py", "vllm_omni/diffusion/models/pi0/processor_pi0.py", "vllm_omni/deploy/pi0.yaml", "tests/diffusion/models/pi0/test_pi0_units.py", "tests/diffusion/models/pi0/test_pi0_parity.py", "tests/e2e/online_serving/test_pi0_expansion.py"] @@ -26,3 +26,10 @@ confidence: high - 验收：CPU unit tests 覆盖 config、camera padding、mask、normalization 与 loader remap；固定 noise 的 CPU/float32 LeRobot parity 保持 `max|Δ| < 1e-4`，pipeline e2e 返回有限的 `[50, 32]` actions，OpenPI websocket handshake 与动作 chunk 通过独立验证。^[PR #4222] 相关页面：[Pi0 目录](./_index.md)；[Diffusion 输出与 multiprocess runtime 规则](../../components/diffusion/rules-output-lifecycle.md)；[配置开发门禁](../../components/configuration/rules.md)。 + +## PI0-2a — OpenPI `nd` 标记必须同时接受 vLLM-native 与 msgpack-numpy 的 `kind` + +- 触发：修改 `entrypoints/openpi/connection.py` 的 `_decode_vllm_numpy_marker` / `_unpack_numpy`，或 OpenPI/DreamZero 观测 payload 解码。 +- 强制：`kind` 仍为区分 array marker 与普通 mapping 的必填字段。plain ndarray 的 `kind` 接受 `""`（msgpack-numpy 包）或 dtype kind 字符（vLLM-native）；二者任一匹配即通过。`kind == "V"`（structured）必须在调用 `np.dtype(type)` 之前拒绝，因该方言的 `type` 是 descriptor list。outbound 仍发 openpi-client 标记。 +- 禁止：要求 `kind == dtype.kind` 而拒绝空 kind；对 structured marker 让 `np.dtype` 抛出不透明 `TypeError`；把 msgpack-numpy **标量**（可省略 `kind`）误当成必须解码的 ndarray marker。 +- 验收：分别覆盖 vLLM-native `kind`、手写 `kind=b""`、真实 `msgpack_numpy.packb` 观测 round-trip、structured `kind=V` 拒绝，以及无 `kind` 的用户 dict 原样保留。^[PR #6051]