Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

launch: a setup option error ('setup.py: error: argument --kv: invalid choice') is the cause shown, not only 'could not be installed'; setup accepts --kv fp16 as DOCKER.md says (#3785); tests

Niko1221bdaa93edf07a
diff --git a/setup.py b/setup.py
index 47e3fa48f..f3689cac0 100644
--- a/setup.py
+++ b/setup.py
@@ -6978,6 +6978,7 @@ UD_FIT_MIN_SLACK_MIB = 64 # at least a few cache slots beside the estima
UD_FIT_CARD_GB = 9.5 # the check is for cards up to here
UD_FIT_RUNTIME_MIB = 250 # CUDA / HIP: the runtime's own context (the SYCL measurement above has none of it)
UD_FIT_ENGINE_RESERVE_MIB = 300 # the reserve the engine settles on when none is given and the card is tight
+KV_WORDS = {"int8": "8-bit", "q4_0": "4-bit (Hadamard-rotated)", "k8v4": "8-bit keys, 4-bit values", "fp16": "16-bit"}
UD_FIT_KV_BYTES = {"fp16": 2112, "int8": 1056, "q4_0": 576, "k8v4": 816} # per token per attention layer (13)
UD_FIT_CTX_STEPS = (8192, 6144, 4608)
@@ -8436,9 +8437,9 @@ def main() -> int:
ap.add_argument("--rope-scale", type=float,
help="the extension factor (default: the final context over the trained 262144, at least 1 - "
"1.5 for 384K, 2 for 512K, 1 inside the trained range)")
- ap.add_argument("--kv", choices=["int8", "q4_0", "k8v4"],
+ ap.add_argument("--kv", choices=["int8", "q4_0", "k8v4", "fp16"],
help="KV cache precision above 8K context: int8 (default), q4_0 (half the memory, a little less "
- "precise) or k8v4 (hybrid: INT8 K + 4-bit V, 816 B/cell)")
+ "precise) k8v4 (hybrid: INT8 K + 4-bit V, 816 B/cell) or fp16 (16-bit, twice the memory of int8)")
ap.add_argument("--vision", choices=["yes", "no", "none", "gpu", "cpu"],
help="let the model read images (on by default where the model can; yes = the best reader for this card: the CPU "
"under 24 GB of VRAM, the GPU from 24 GB, any make; gpu / cpu = that reader; no = text only)")
@@ -9105,7 +9106,7 @@ def main() -> int:
say(" documents; long-context lookups (needle tests) still pass")
kv = ["int8", "q4_0"][int(ask("KV cache?", ["1", "2"], "1", a.yes)) - 1]
if ctx > 8192:
- ok(f"KV cache: {'8-bit' if kv == 'int8' else '4-bit (Hadamard-rotated)'}")
+ ok(f"KV cache: {KV_WORDS.get(kv, kv)}")
if MODELS[model].get("vision", fam.get("vision")) is False: # UD-IQ4_XS: images, unlike UD-Q4_K_XL
vision = "none"
if a.vision not in (None, "no", "none"):
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index a09f6b8fd..e02a20324 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -77,6 +77,16 @@ def python_failure(lines: list[str]) -> str | None:
return None
+def argparse_failure(lines: list[str]) -> str:
+ """setup.py's own refusal of an option ("setup.py: error: argument --kv: invalid choice: ..."), which is neither an [X] line nor a
+ traceback (#3785): the words after "error:" are the cause."""
+ for line in reversed(lines):
+ found = re.match(r"\s*(?:setup|strata)(?:\.py)?: error: (.+)", line)
+ if found:
+ return found.group(1).strip()[:300]
+ return ""
+
+
def missing_file(failure: str) -> bool:
"""An import or file error: the install itself is short of a file, which an update of Strata brings back."""
return failure.startswith(("ModuleNotFoundError", "ImportError", "FileNotFoundError"))
@@ -99,6 +109,7 @@ def setup_failure(lines: list[str], err: str, hint: str, details: str | None, ow
"""The error for a setup.py run that exited non-zero. setup.py's own `[X]` line (err) is the message when there is one; when it
stopped with a Python error instead, that error's last line is (#3613, #3467: a file missing from the Terminal's tools/ ended every
launch after a tune with only "The model could not be installed"). `details` is the file that holds the rest of its output."""
+ err = err or argparse_failure(lines)
crashed = python_failure(lines) if not err else None
if crashed:
where = f" The whole setup output is in {details}." if details else ""
diff --git a/tests/terminal/test_setup_failure_words.py b/tests/terminal/test_setup_failure_words.py
index 1cf70ec66..7d111714f 100644
--- a/tests/terminal/test_setup_failure_words.py
+++ b/tests/terminal/test_setup_failure_words.py
@@ -18,6 +18,20 @@ TRACEBACK = [
class SetupFailureWords(unittest.TestCase):
+ def test_an_option_setup_refuses_is_said_with_its_reason(self): # #3785
+ lines = ["Preparing unsloth UD-Q6_K_XL from your model files...",
+ "setup.py: error: argument --kv: invalid choice: 'bf16' (choose from int8, q4_0, k8v4, fp16)"]
+ error = setup_failure(lines, "", "", "/tmp/a.setup.log", True)
+ self.assertIn("The model could not be installed. Argument --kv: invalid choice: 'bf16'", error.text)
+ self.assertIn("/tmp/a.setup.log", error.next)
+ self.assertEqual(setup_failure(["nothing useful"], "", "", None, True).text, "The model could not be installed.")
+
+ def test_setup_accepts_the_kv_format_the_docker_docs_offer(self):
+ import subprocess, sys
+ root = Path(__file__).resolve().parents[2]
+ done = subprocess.run([sys.executable, str(root / "setup.py"), "--kv", "bogus"], capture_output=True, text=True, cwd=str(root), timeout=120)
+ self.assertIn("choose from int8, q4_0, k8v4, fp16", done.stderr)
+
def test_the_last_line_of_a_traceback_is_found(self):
self.assertEqual(python_failure(TRACEBACK), "ModuleNotFoundError: No module named 'calibrate'")
self.assertIsNone(python_failure(["[X] Not enough disk space", " Free some space"]))