Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
launch: a setup option error ('setup.py: error: argument --kv: invalid choice') is the cause shown, not only 'could not be installed'; setup accepts --kv fp16 as DOCKER.md says (#3785); tests
Niko1221bdaa93edf07a
diff --git a/setup.py b/setup.pyindex 47e3fa48f..f3689cac0 100644--- a/setup.py+++ b/setup.py@@ -6978,6 +6978,7 @@ UD_FIT_MIN_SLACK_MIB = 64 # at least a few cache slots beside the estimaUD_FIT_CARD_GB = 9.5 # the check is for cards up to hereUD_FIT_RUNTIME_MIB = 250 # CUDA / HIP: the runtime's own context (the SYCL measurement above has none of it)UD_FIT_ENGINE_RESERVE_MIB = 300 # the reserve the engine settles on when none is given and the card is tight+KV_WORDS = {"int8": "8-bit", "q4_0": "4-bit (Hadamard-rotated)", "k8v4": "8-bit keys, 4-bit values", "fp16": "16-bit"}UD_FIT_KV_BYTES = {"fp16": 2112, "int8": 1056, "q4_0": 576, "k8v4": 816} # per token per attention layer (13)UD_FIT_CTX_STEPS = (8192, 6144, 4608)@@ -8436,9 +8437,9 @@ def main() -> int:ap.add_argument("--rope-scale", type=float,help="the extension factor (default: the final context over the trained 262144, at least 1 - ""1.5 for 384K, 2 for 512K, 1 inside the trained range)")- ap.add_argument("--kv", choices=["int8", "q4_0", "k8v4"],+ ap.add_argument("--kv", choices=["int8", "q4_0", "k8v4", "fp16"],help="KV cache precision above 8K context: int8 (default), q4_0 (half the memory, a little less "- "precise) or k8v4 (hybrid: INT8 K + 4-bit V, 816 B/cell)")+ "precise) k8v4 (hybrid: INT8 K + 4-bit V, 816 B/cell) or fp16 (16-bit, twice the memory of int8)")ap.add_argument("--vision", choices=["yes", "no", "none", "gpu", "cpu"],help="let the model read images (on by default where the model can; yes = the best reader for this card: the CPU ""under 24 GB of VRAM, the GPU from 24 GB, any make; gpu / cpu = that reader; no = text only)")@@ -9105,7 +9106,7 @@ def main() -> int:say(" documents; long-context lookups (needle tests) still pass")kv = ["int8", "q4_0"][int(ask("KV cache?", ["1", "2"], "1", a.yes)) - 1]if ctx > 8192:- ok(f"KV cache: {'8-bit' if kv == 'int8' else '4-bit (Hadamard-rotated)'}")+ ok(f"KV cache: {KV_WORDS.get(kv, kv)}")if MODELS[model].get("vision", fam.get("vision")) is False: # UD-IQ4_XS: images, unlike UD-Q4_K_XLvision = "none"if a.vision not in (None, "no", "none"):diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.pyindex a09f6b8fd..e02a20324 100644--- a/strata_cli/launch/runner.py+++ b/strata_cli/launch/runner.py@@ -77,6 +77,16 @@ def python_failure(lines: list[str]) -> str | None:return None+def argparse_failure(lines: list[str]) -> str:+ """setup.py's own refusal of an option ("setup.py: error: argument --kv: invalid choice: ..."), which is neither an [X] line nor a+ traceback (#3785): the words after "error:" are the cause."""+ for line in reversed(lines):+ found = re.match(r"\s*(?:setup|strata)(?:\.py)?: error: (.+)", line)+ if found:+ return found.group(1).strip()[:300]+ return ""++def missing_file(failure: str) -> bool:"""An import or file error: the install itself is short of a file, which an update of Strata brings back."""return failure.startswith(("ModuleNotFoundError", "ImportError", "FileNotFoundError"))@@ -99,6 +109,7 @@ def setup_failure(lines: list[str], err: str, hint: str, details: str | None, ow"""The error for a setup.py run that exited non-zero. setup.py's own `[X]` line (err) is the message when there is one; when itstopped with a Python error instead, that error's last line is (#3613, #3467: a file missing from the Terminal's tools/ ended everylaunch after a tune with only "The model could not be installed"). `details` is the file that holds the rest of its output."""+ err = err or argparse_failure(lines)crashed = python_failure(lines) if not err else Noneif crashed:where = f" The whole setup output is in {details}." if details else ""diff --git a/tests/terminal/test_setup_failure_words.py b/tests/terminal/test_setup_failure_words.pyindex 1cf70ec66..7d111714f 100644--- a/tests/terminal/test_setup_failure_words.py+++ b/tests/terminal/test_setup_failure_words.py@@ -18,6 +18,20 @@ TRACEBACK = [class SetupFailureWords(unittest.TestCase):+ def test_an_option_setup_refuses_is_said_with_its_reason(self): # #3785+ lines = ["Preparing unsloth UD-Q6_K_XL from your model files...",+ "setup.py: error: argument --kv: invalid choice: 'bf16' (choose from int8, q4_0, k8v4, fp16)"]+ error = setup_failure(lines, "", "", "/tmp/a.setup.log", True)+ self.assertIn("The model could not be installed. Argument --kv: invalid choice: 'bf16'", error.text)+ self.assertIn("/tmp/a.setup.log", error.next)+ self.assertEqual(setup_failure(["nothing useful"], "", "", None, True).text, "The model could not be installed.")++ def test_setup_accepts_the_kv_format_the_docker_docs_offer(self):+ import subprocess, sys+ root = Path(__file__).resolve().parents[2]+ done = subprocess.run([sys.executable, str(root / "setup.py"), "--kv", "bogus"], capture_output=True, text=True, cwd=str(root), timeout=120)+ self.assertIn("choose from int8, q4_0, k8v4, fp16", done.stderr)+def test_the_last_line_of_a_traceback_is_found(self):self.assertEqual(python_failure(TRACEBACK), "ModuleNotFoundError: No module named 'calibrate'")self.assertIsNone(python_failure(["[X] Not enough disk space", " Free some space"]))