Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Fix #3502: a CUDA-init failure in a tuning run is not recorded as OOM (the memory guard info line no longer decides), and a command that passed before is not blacklisted by a later failure

Niko12216515aebac2a6
diff --git a/strata_control/presets.py b/strata_control/presets.py
index ad95c940e..2ab4f0016 100644
--- a/strata_control/presets.py
+++ b/strata_control/presets.py
@@ -508,14 +508,19 @@ def record_trials(trials: list[dict[str, Any]], *, hw_hash: str, model_digest: s
def bad_signatures(*, hw_hash: str, model_digest: str | None, engine_sha: str | None) -> set[str]:
- """Candidates that did not start on this PC with this engine and model before (UNSUPPORTED or OOM): never retried."""
+ """Candidates that did not start on this PC with this engine and model before (UNSUPPORTED or OOM): never retried.
+ A candidate that also has a PASS row for the same command here is not listed: one busy-card failure after it
+ worked must not switch it off for good (#3502)."""
bad: set[str] = set()
+ passed: set[str] = set()
try:
for line in history_path().read_text(encoding="utf-8").splitlines():
r = json.loads(line)
- if r.get("hw") == hw_hash and r.get("model") == model_digest and r.get("engine") == engine_sha and \
- r.get("status") in ("UNSUPPORTED", "OOM"):
- bad.add(r.get("label") or "")
+ if r.get("hw") == hw_hash and r.get("model") == model_digest and r.get("engine") == engine_sha:
+ if r.get("status") in ("UNSUPPORTED", "OOM"):
+ bad.add(r.get("label") or "")
+ elif r.get("status") == "PASS":
+ passed.add(r.get("label") or "")
except (OSError, ValueError):
pass
- return bad
+ return bad - passed
diff --git a/strata_control/tune_session.py b/strata_control/tune_session.py
index be82408a9..b32ad8063 100644
--- a/strata_control/tune_session.py
+++ b/strata_control/tune_session.py
@@ -126,15 +126,24 @@ def parse_facts(text: str) -> dict[str, Any]:
return facts
+_CUDA_BUSY = re.compile(r"failed to initialize cuda|no cuda-capable device|cuda-capable device\(s\) is/are busy|"
+ r"cuda driver version is insufficient|hipinit.{0,40}(?:fail|error)|no hip-capable device", re.I)
+
+
def failure_kind(text: str, returncode: int | None) -> tuple[str, str]:
- """Classify an engine that ended before it was ready: (status, one plain line)."""
+ """Classify an engine that ended before it was ready: (status, one plain line). The engine's informational
+ `memory guard: limit ... available ...` line is not a failure and never decides the status (#3502)."""
low = text.lower()
- lines = [ln.strip() for ln in text.splitlines() if re.search(
- r"unknown argument|unrecognized|invalid value|out of memory|memory guard|cannot|failed|error|refus|killed", ln, re.I)]
+ lines = [ln.strip() for ln in text.splitlines()
+ if re.search(r"unknown argument|unrecognized|invalid value|out of memory|cannot|failed|error|refus|killed", ln, re.I)
+ and not re.search(r"memory guard: (?:limit|the available)", ln, re.I)]
reason = " | ".join(lines[:3])[:300]
if re.search(r"unknown argument|unrecognized (argument|option)", low):
return "UNSUPPORTED", reason or "the engine does not accept one of the settings"
- if re.search(r"out of memory|memory guard|cannot allocate|std::bad_alloc|\boom\b|(?:^|\n)killed(?:\r?\n|$)", low) or returncode in (-9, 137, 3221226505):
+ if _CUDA_BUSY.search(text):
+ return "FAIL_BOOT", ("the graphics card could not be opened by the measuring engine; it is probably in use by the "
+ "loaded model. Unload the model and tune again (" + reason + ")")[:400]
+ if re.search(r"out of memory|memory guard: (?:refus|not enough|too)|cannot allocate|std::bad_alloc|\boom\b|(?:^|\n)killed(?:\r?\n|$)", low) or returncode in (-9, 137, 3221226505):
return "OOM", reason or "the engine ran out of memory"
return "FAIL_BOOT", reason or f"the engine stopped while starting (exit {returncode})"
diff --git a/tests/tuning/test_presets.py b/tests/tuning/test_presets.py
index ab46de518..98b6db06a 100644
--- a/tests/tuning/test_presets.py
+++ b/tests/tuning/test_presets.py
@@ -170,6 +170,13 @@ class StoreTest(unittest.TestCase):
self.assertEqual(presets.bad_signatures(hw_hash="h2", model_digest="m", engine_sha="e"), set())
self.assertEqual(presets.bad_signatures(hw_hash="h", model_digest="m", engine_sha="e2"), set())
+ def test_a_command_that_passed_is_not_switched_off_by_a_later_failure(self):
+ # #3502: the plain base command passed four times, then a busy card wrote OOM rows
+ for status in ("PASS", "OOM"):
+ presets.record_trials([{"label": "base", "args": ["--x"], "env": {}, "status": status, "at": 1}],
+ hw_hash="h", model_digest="m", engine_sha="e")
+ self.assertEqual(presets.bad_signatures(hw_hash="h", model_digest="m", engine_sha="e"), set())
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/tuning/test_session.py b/tests/tuning/test_session.py
index 3474b801a..7aa391f56 100644
--- a/tests/tuning/test_session.py
+++ b/tests/tuning/test_session.py
@@ -104,6 +104,19 @@ class ParsingTest(unittest.TestCase):
self.assertEqual(failure_kind("", -9)[0], "OOM")
self.assertEqual(failure_kind("something else broke", 1)[0], "FAIL_BOOT")
+ def test_cuda_init_failure_with_the_memory_guard_line_is_not_oom(self):
+ # #3502: the informational memory guard line was present in every failed run
+ text = ("ggml_cuda_init: failed to initialize CUDA: no CUDA-capable device is detected\n"
+ "strata generate: memory guard: limit 63.8 GiB (source: meminfo), current 12.6 GiB, reclaimable cache "
+ "credited 0.0 GiB; available 51.2 GiB (MemAvailable 51.2 GiB), floor 2560 MiB\n"
+ "strata generate: expert arena: cudaHostRegister of the whole arena FAILED (out of memory)\n")
+ status, why = failure_kind(text, 1)
+ self.assertEqual(status, "FAIL_BOOT")
+ self.assertIn("Unload the model", why)
+ self.assertNotIn("memory guard", why)
+ info = "strata generate: memory guard: limit 63.8 GiB, available 51.2 GiB\nsomething else broke"
+ self.assertEqual(failure_kind(info, 1)[0], "FAIL_BOOT")
+
def test_gpu_cache_wording_is_not_the_low_ram_mode(self):
self.assertFalse(parse_facts("R4 hit path ON - resident experts are computed on the GPU")["resident_ram"])
self.assertTrue(parse_facts("WARNING: the whole resident RAM mode does not fit (FileExpertSource: resident complement 34.47 GiB")["resident_ram"])