Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

serve: opt-in LaTeX-safe JSON recovery for structured output (#2893)

Niko12212cc6ea169b91
diff --git a/docs/DETAILS.md b/docs/DETAILS.md
index 0ed983c37..0f16b8be6 100644
--- a/docs/DETAILS.md
+++ b/docs/DETAILS.md
@@ -492,7 +492,11 @@ applies at the next start. Network, key, MCP and program keys are not editable t
decoding, with one generation per request. `jsonschema` is used when installed (`python -m pip install "jsonschema>=4.23,<5"`);
without it the answer is only checked to be one JSON object. Malformed output returns **502** with
`error.code: structured_output_failed`, an invalid schema **400**. JSON formats combined with tools or MCP are refused.
- Streaming buffers the answer and emits content only after validation.
+ Streaming buffers the answer and emits content only after validation. Math answers often hold LaTeX such as `\alpha`, which is
+ not a valid JSON escape. Add `"repair_latex_escapes": true` to the `response_format` (or to `text.format` on `/v1/responses`) and,
+ only after the strict parse failed, the server doubles the backslashes that are not valid escapes and checks the answer again.
+ Valid output is never touched. `\b`, `\f`, `\n`, `\r`, `\t` and `\uXXXX` are valid JSON, so they stay: a `\beta` beside a `\alpha`
+ comes back with a backspace. Off by default.
- `logit_bias` and token/bias pair lists work on CUDA and HIP engines without continuous batching ([LOGIT_BIAS.md](LOGIT_BIAS.md)).
- Stop strings (`stop`, `stop_sequences`), `temperature`, `top_p`, `top_k`, `min_p` and `seed` are honored per request.
`top_k` keeps at most 64 candidates. Penalties (`presence_penalty`, `frequency_penalty`, `repetition_penalty`,
diff --git a/serve/responses.py b/serve/responses.py
index a5b97bce1..d5ea4cd00 100644
--- a/serve/responses.py
+++ b/serve/responses.py
@@ -420,11 +420,12 @@ def text_format(req: dict):
kind = fmt.get("type")
if kind in (None, "text"):
return None
+ extra = {"repair_latex_escapes": True} if fmt.get("repair_latex_escapes") is True else {} # #2893, opt-in
if kind == "json_object":
- return {"type": "json_object"}
+ return {"type": "json_object", **extra}
if kind == "json_schema":
spec = {k: fmt[k] for k in ("name", "schema", "strict", "description") if k in fmt}
- return {"type": "json_schema", "json_schema": spec}
+ return {"type": "json_schema", "json_schema": spec, **extra}
raise ResponsesError("text.format.type must be text, json_object or json_schema", "text.format.type")
diff --git a/serve/structured.py b/serve/structured.py
index 9bed53484..99ecf1c81 100644
--- a/serve/structured.py
+++ b/serve/structured.py
@@ -28,6 +28,46 @@ class _ObjectError:
message = "the answer is not a JSON object"
+class _Repairing:
+ """A validator that also tells validated_json to retry once, after a strict parse failed, with the backslashes that
+ are not valid JSON escapes doubled (response_format "repair_latex_escapes": true, #2893)."""
+ repair_escapes = True
+
+ def __init__(self, inner):
+ self.inner = inner
+
+ def iter_errors(self, value):
+ return self.inner.iter_errors(value)
+
+
+_HEX = "0123456789abcdefABCDEF"
+
+
+def repair_latex_escapes(text: str) -> str:
+ r"""Double every backslash that does not start a valid JSON escape, so \alpha and \gamma read as text.
+ The valid escapes (\b \f \n \r \t \/ \" and a well-formed \uXXXX) stay as they are: a model cannot be told
+ from LaTeX there, so a \beta next to a \alpha comes back with a backspace. Pairs of backslashes are skipped."""
+ out = []
+ i, n = 0, len(text)
+ while i < n:
+ ch = text[i]
+ if ch != "\\":
+ out.append(ch)
+ i += 1
+ continue
+ nxt = text[i + 1] if i + 1 < n else ""
+ if nxt in ("\\", '"', "/", "b", "f", "n", "r", "t"):
+ out.append(ch + nxt)
+ i += 2
+ elif nxt == "u" and len(text) >= i + 6 and all(c in _HEX for c in text[i + 2:i + 6]):
+ out.append(text[i:i + 6])
+ i += 6
+ else:
+ out.append("\\\\")
+ i += 1
+ return "".join(out)
+
+
_jsonschema = None # (validators, SchemaError, Registry, NoSuchResource), False when not installed
_jsonschema_lock = threading.Lock()
@@ -140,6 +180,8 @@ def prepare_format(response_format, messages, with_tools=False):
# /v1/responses (#782): the schema is for the final answer; a turn that calls a tool is not an answer
directive += ("\nThis applies only to your final answer. To use a tool, call it as usual; the JSON object is "
"what you write once you are done with the tools.")
+ if response_format.get("repair_latex_escapes") is True:
+ validator = _Repairing(validator)
messages = [dict(message) for message in messages]
if messages and messages[0].get("role") == "system":
content = messages[0].get("content") or ""
@@ -207,7 +249,17 @@ def validated_json(text, validator, finish):
value = json.loads(_extract_json(text), object_pairs_hook=pairs, parse_constant=constant)
canonical = json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
except (ValueError, TypeError) as exc:
- raise StructuredOutputError(f"model did not return valid JSON: {exc}") from exc
+ value = None
+ if getattr(validator, "repair_escapes", False): # opt-in (#2893); only after the strict parse failed
+ try:
+ value = json.loads(repair_latex_escapes(_extract_json(text)), object_pairs_hook=pairs,
+ parse_constant=constant)
+ canonical = json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
+ print("[strata] structured output: repaired backslashes that were not valid JSON escapes", flush=True)
+ except (ValueError, TypeError):
+ value = None
+ if value is None:
+ raise StructuredOutputError(f"model did not return valid JSON: {exc}") from exc
try:
error = next(validator.iter_errors(value), None)
except Exception as exc:
diff --git a/serve/test_structured.py b/serve/test_structured.py
index 6b66cf8b1..f0e2c74ae 100644
--- a/serve/test_structured.py
+++ b/serve/test_structured.py
@@ -112,6 +112,41 @@ class Structured(unittest.TestCase):
# Plain text callers retain normal streaming, without a JSON validation gate.
self.assertEqual(self.chat("Prose.", response_format={"type": "text"})[0], 200)
+ def test_latex_escape_repair_is_opt_in_and_only_after_a_strict_failure(self):
+ # #2893: \alpha is not a JSON escape. Off by default: still a 502. On: one retry after the strict parse failed.
+ bad = r'{"answer":"the angle \alpha and \gamma_0"}'
+ self.assertEqual(self.chat(bad, response_format={"type": "json_object"})[0], 502)
+ self.assertEqual(self.chat(bad, response_format={"type": "json_object", "repair_latex_escapes": False})[0], 502)
+ code, reply = self.chat(bad, response_format={"type": "json_object", "repair_latex_escapes": True})
+ self.assertEqual(code, 200)
+ self.assertEqual(json.loads(reply["choices"][0]["message"]["content"]),
+ {"answer": "the angle \\alpha and \\gamma_0"})
+ # streaming asks for the same check before any content leaves
+ _, raw = self.chat(bad, stream=True, response_format={"type": "json_object", "repair_latex_escapes": True})
+ self.assertIn(r"\\alpha", raw)
+ # the repair does not hide what is wrong with the answer itself
+ fmt = {"type": "json_object", "repair_latex_escapes": True}
+ for script in ('[]', r'{"x":NaN}', r'{"a":"\alpha","a":2}', 'no json at all', r'{"x":"\alpha'):
+ self.assertEqual(self.chat(script, response_format=fmt)[0], 502, script)
+ self.assertEqual(self.chat(r'{"answer":"\alpha"}', response_format=fmt, max_tokens=5)[0], 502)
+
+ def test_latex_escape_repair_helper(self):
+ fix = structured.repair_latex_escapes
+ self.assertEqual(fix(r'{"a":"\alpha \gamma_0"}'), r'{"a":"\\alpha \\gamma_0"}')
+ for same in (r'{"a":"x\ny\t\"q\" \\ \/ \u00e9"}', '{"a": 1}', ''):
+ self.assertEqual(fix(same), same)
+ self.assertEqual(fix(r'{"a":"\u12"}'), r'{"a":"\\u12"}') # \u without four hex digits
+ self.assertEqual(fix(r'{"a":"\\alpha"}'), r'{"a":"\\alpha"}') # an escaped backslash is skipped as a pair
+ self.assertEqual(fix("\\"), "\\\\") # a lone trailing backslash
+ # \b is a valid escape: untouched, so it parses (as a backspace), as it does without the option
+ self.assertEqual(json.loads(fix(r'{"a":"\beta \alpha"}'))["a"], "\beta \\alpha")
+
+ def test_the_repair_option_reaches_the_responses_format(self):
+ from serve.responses import text_format
+ self.assertEqual(text_format({"text": {"format": {"type": "json_object", "repair_latex_escapes": True}}}),
+ {"type": "json_object", "repair_latex_escapes": True})
+ self.assertEqual(text_format({"text": {"format": {"type": "json_object"}}}), {"type": "json_object"})
+
def test_rejected_schema_does_not_load_the_engine_or_retry(self):
with mock.patch.object(self.svc, "load") as load:
code, _ = self.chat(response_format={"type": "json_schema"})