Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
serve: opt-in LaTeX-safe JSON recovery for structured output (#2893)
Niko12212cc6ea169b91
diff --git a/docs/DETAILS.md b/docs/DETAILS.mdindex 0ed983c37..0f16b8be6 100644--- a/docs/DETAILS.md+++ b/docs/DETAILS.md@@ -492,7 +492,11 @@ applies at the next start. Network, key, MCP and program keys are not editable tdecoding, with one generation per request. `jsonschema` is used when installed (`python -m pip install "jsonschema>=4.23,<5"`);without it the answer is only checked to be one JSON object. Malformed output returns **502** with`error.code: structured_output_failed`, an invalid schema **400**. JSON formats combined with tools or MCP are refused.- Streaming buffers the answer and emits content only after validation.+ Streaming buffers the answer and emits content only after validation. Math answers often hold LaTeX such as `\alpha`, which is+ not a valid JSON escape. Add `"repair_latex_escapes": true` to the `response_format` (or to `text.format` on `/v1/responses`) and,+ only after the strict parse failed, the server doubles the backslashes that are not valid escapes and checks the answer again.+ Valid output is never touched. `\b`, `\f`, `\n`, `\r`, `\t` and `\uXXXX` are valid JSON, so they stay: a `\beta` beside a `\alpha`+ comes back with a backspace. Off by default.- `logit_bias` and token/bias pair lists work on CUDA and HIP engines without continuous batching ([LOGIT_BIAS.md](LOGIT_BIAS.md)).- Stop strings (`stop`, `stop_sequences`), `temperature`, `top_p`, `top_k`, `min_p` and `seed` are honored per request.`top_k` keeps at most 64 candidates. Penalties (`presence_penalty`, `frequency_penalty`, `repetition_penalty`,diff --git a/serve/responses.py b/serve/responses.pyindex a5b97bce1..d5ea4cd00 100644--- a/serve/responses.py+++ b/serve/responses.py@@ -420,11 +420,12 @@ def text_format(req: dict):kind = fmt.get("type")if kind in (None, "text"):return None+ extra = {"repair_latex_escapes": True} if fmt.get("repair_latex_escapes") is True else {} # #2893, opt-inif kind == "json_object":- return {"type": "json_object"}+ return {"type": "json_object", **extra}if kind == "json_schema":spec = {k: fmt[k] for k in ("name", "schema", "strict", "description") if k in fmt}- return {"type": "json_schema", "json_schema": spec}+ return {"type": "json_schema", "json_schema": spec, **extra}raise ResponsesError("text.format.type must be text, json_object or json_schema", "text.format.type")diff --git a/serve/structured.py b/serve/structured.pyindex 9bed53484..99ecf1c81 100644--- a/serve/structured.py+++ b/serve/structured.py@@ -28,6 +28,46 @@ class _ObjectError:message = "the answer is not a JSON object"+class _Repairing:+ """A validator that also tells validated_json to retry once, after a strict parse failed, with the backslashes that+ are not valid JSON escapes doubled (response_format "repair_latex_escapes": true, #2893)."""+ repair_escapes = True++ def __init__(self, inner):+ self.inner = inner++ def iter_errors(self, value):+ return self.inner.iter_errors(value)+++_HEX = "0123456789abcdefABCDEF"+++def repair_latex_escapes(text: str) -> str:+ r"""Double every backslash that does not start a valid JSON escape, so \alpha and \gamma read as text.+ The valid escapes (\b \f \n \r \t \/ \" and a well-formed \uXXXX) stay as they are: a model cannot be told+ from LaTeX there, so a \beta next to a \alpha comes back with a backspace. Pairs of backslashes are skipped."""+ out = []+ i, n = 0, len(text)+ while i < n:+ ch = text[i]+ if ch != "\\":+ out.append(ch)+ i += 1+ continue+ nxt = text[i + 1] if i + 1 < n else ""+ if nxt in ("\\", '"', "/", "b", "f", "n", "r", "t"):+ out.append(ch + nxt)+ i += 2+ elif nxt == "u" and len(text) >= i + 6 and all(c in _HEX for c in text[i + 2:i + 6]):+ out.append(text[i:i + 6])+ i += 6+ else:+ out.append("\\\\")+ i += 1+ return "".join(out)++_jsonschema = None # (validators, SchemaError, Registry, NoSuchResource), False when not installed_jsonschema_lock = threading.Lock()@@ -140,6 +180,8 @@ def prepare_format(response_format, messages, with_tools=False):# /v1/responses (#782): the schema is for the final answer; a turn that calls a tool is not an answerdirective += ("\nThis applies only to your final answer. To use a tool, call it as usual; the JSON object is ""what you write once you are done with the tools.")+ if response_format.get("repair_latex_escapes") is True:+ validator = _Repairing(validator)messages = [dict(message) for message in messages]if messages and messages[0].get("role") == "system":content = messages[0].get("content") or ""@@ -207,7 +249,17 @@ def validated_json(text, validator, finish):value = json.loads(_extract_json(text), object_pairs_hook=pairs, parse_constant=constant)canonical = json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))except (ValueError, TypeError) as exc:- raise StructuredOutputError(f"model did not return valid JSON: {exc}") from exc+ value = None+ if getattr(validator, "repair_escapes", False): # opt-in (#2893); only after the strict parse failed+ try:+ value = json.loads(repair_latex_escapes(_extract_json(text)), object_pairs_hook=pairs,+ parse_constant=constant)+ canonical = json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))+ print("[strata] structured output: repaired backslashes that were not valid JSON escapes", flush=True)+ except (ValueError, TypeError):+ value = None+ if value is None:+ raise StructuredOutputError(f"model did not return valid JSON: {exc}") from exctry:error = next(validator.iter_errors(value), None)except Exception as exc:diff --git a/serve/test_structured.py b/serve/test_structured.pyindex 6b66cf8b1..f0e2c74ae 100644--- a/serve/test_structured.py+++ b/serve/test_structured.py@@ -112,6 +112,41 @@ class Structured(unittest.TestCase):# Plain text callers retain normal streaming, without a JSON validation gate.self.assertEqual(self.chat("Prose.", response_format={"type": "text"})[0], 200)+ def test_latex_escape_repair_is_opt_in_and_only_after_a_strict_failure(self):+ # #2893: \alpha is not a JSON escape. Off by default: still a 502. On: one retry after the strict parse failed.+ bad = r'{"answer":"the angle \alpha and \gamma_0"}'+ self.assertEqual(self.chat(bad, response_format={"type": "json_object"})[0], 502)+ self.assertEqual(self.chat(bad, response_format={"type": "json_object", "repair_latex_escapes": False})[0], 502)+ code, reply = self.chat(bad, response_format={"type": "json_object", "repair_latex_escapes": True})+ self.assertEqual(code, 200)+ self.assertEqual(json.loads(reply["choices"][0]["message"]["content"]),+ {"answer": "the angle \\alpha and \\gamma_0"})+ # streaming asks for the same check before any content leaves+ _, raw = self.chat(bad, stream=True, response_format={"type": "json_object", "repair_latex_escapes": True})+ self.assertIn(r"\\alpha", raw)+ # the repair does not hide what is wrong with the answer itself+ fmt = {"type": "json_object", "repair_latex_escapes": True}+ for script in ('[]', r'{"x":NaN}', r'{"a":"\alpha","a":2}', 'no json at all', r'{"x":"\alpha'):+ self.assertEqual(self.chat(script, response_format=fmt)[0], 502, script)+ self.assertEqual(self.chat(r'{"answer":"\alpha"}', response_format=fmt, max_tokens=5)[0], 502)++ def test_latex_escape_repair_helper(self):+ fix = structured.repair_latex_escapes+ self.assertEqual(fix(r'{"a":"\alpha \gamma_0"}'), r'{"a":"\\alpha \\gamma_0"}')+ for same in (r'{"a":"x\ny\t\"q\" \\ \/ \u00e9"}', '{"a": 1}', ''):+ self.assertEqual(fix(same), same)+ self.assertEqual(fix(r'{"a":"\u12"}'), r'{"a":"\\u12"}') # \u without four hex digits+ self.assertEqual(fix(r'{"a":"\\alpha"}'), r'{"a":"\\alpha"}') # an escaped backslash is skipped as a pair+ self.assertEqual(fix("\\"), "\\\\") # a lone trailing backslash+ # \b is a valid escape: untouched, so it parses (as a backspace), as it does without the option+ self.assertEqual(json.loads(fix(r'{"a":"\beta \alpha"}'))["a"], "\beta \\alpha")++ def test_the_repair_option_reaches_the_responses_format(self):+ from serve.responses import text_format+ self.assertEqual(text_format({"text": {"format": {"type": "json_object", "repair_latex_escapes": True}}}),+ {"type": "json_object", "repair_latex_escapes": True})+ self.assertEqual(text_format({"text": {"format": {"type": "json_object"}}}), {"type": "json_object"})+def test_rejected_schema_does_not_load_the_engine_or_retry(self):with mock.patch.object(self.svc, "load") as load:code, _ = self.chat(response_format={"type": "json_schema"})