ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
perf(catalog): skip GLM's FlashInfer autotune, which bought nothing (#8295)
Mike Bradleye6bf5ea519d9
diff --git a/ods/config/model-library.json b/ods/config/model-library.jsonindex aa3358627..1806a478c 100644--- a/ods/config/model-library.json+++ b/ods/config/model-library.json@@ -2906,7 +2906,7 @@"reasoning": "medium","tool_call_parser": "glm47","reasoning_parser": "glm47",- "vllm_args": ["--max-model-len", "131072", "--kv-cache-dtype", "fp8", "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--enable-auto-tool-choice", "--tool-call-parser", "glm47", "--reasoning-parser", "glm47", "--trust-remote-code", "--speculative-config.method=mtp", "--speculative-config.num_speculative_tokens=5"],+ "vllm_args": ["--max-model-len", "131072", "--kv-cache-dtype", "fp8", "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--enable-prefix-caching", "--enable-auto-tool-choice", "--tool-call-parser", "glm47", "--reasoning-parser", "glm47", "--trust-remote-code", "--speculative-config.method=mtp", "--speculative-config.num_speculative_tokens=5", "--no-enable-flashinfer-autotune"],"selection": {"discrete": 0,"unified": 0,diff --git a/ods/tests/test-model-library-vllm.py b/ods/tests/test-model-library-vllm.pyindex bb6ad8866..d29d638c4 100644--- a/ods/tests/test-model-library-vllm.py+++ b/ods/tests/test-model-library-vllm.py@@ -193,11 +193,24 @@ MEASURED_FLAGS = {}+# Plain switches measured the same way. The FlashInfer autotune loaded all 63 cached configs and+# still ran its dummy passes for about 4 minutes on every GLM start; off, a restart was ready in+# 156 s instead of 392 s at the same speed (1/4/16 streams: 349/965/2,276 tok/s against 349/961/2,246).+# GLM fits only sm_103 GPUs (GB300/B300), where that was measured. Qwen3.8 also showed no loss on the+# GB300 (226 s -> 155 s), but its main fit is B200-class sm_100, which is unmeasured, so it keeps it.+MEASURED_SWITCHES = {+ "qwen3.8-flash-next-nvfp4-vllm": set(),+ "glm-5.3-flash-nvfp4-vllm": {"--no-enable-flashinfer-autotune"},+}++@pytest.mark.parametrize("model_id", sorted(MEASURED_FLAGS))def test_the_entries_keep_the_flags_measured_on_a_dgx_station(model_id):model = {item["id"]: item for item in _vllm_models()}[model_id]for flag, expected in MEASURED_FLAGS[model_id].items():assert _dotted_config(model["vllm_args"], flag) == expected, (model_id, flag)+ switches = {arg for arg in model["vllm_args"] if arg == "--no-enable-flashinfer-autotune"}+ assert switches == MEASURED_SWITCHES[model_id], model_iddef test_the_broken_entry_cases_edit_what_they_name():