Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
review: docs section, 1 MiB grammar pre-check, negative tests, measured compile overhead
FABIOTESS84514a5f395a
diff --git a/c/openai_server.py b/c/openai_server.pyindex 1dd987ba..ae1032c9 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -468,6 +468,9 @@ def generation_options(body, limit):raise APIError(400, "`response_format.type` must be \"text\", \"json_object\", ""\"json_schema\" or \"gbnf\".","response_format", "unsupported_value")+ if grammar is not None and len(grammar.encode("utf-8")) > (1 << 20):+ raise APIError(400, "`response_format` grammar/schema exceeds 1 MiB.",+ "response_format", "invalid_value")maximum = body.get("max_completion_tokens")maximum_param = "max_completion_tokens"diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 7cbbc045..8000b400 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -99,6 +99,16 @@ class TemplateTest(unittest.TestCase):generation_options({"response_format": {"type": "yaml"}}, 8)with self.assertRaises(APIError):generation_options({"response_format": {"type": "json_schema", "json_schema": {}}}, 8)+ with self.assertRaises(APIError): # non-dict response_format+ generation_options({"response_format": "json"}, 8)+ with self.assertRaises(APIError): # empty gbnf+ generation_options({"response_format": {"type": "gbnf", "grammar": " "}}, 8)+ with self.assertRaises(APIError): # oversized grammar (> 1 MiB pre-check)+ generation_options({"response_format": {"type": "gbnf", "grammar": "x" * ((1 << 20) + 1)}}, 8)+ # malformed GBNF passes the gateway by design: the ENGINE fail-softs it+ # (draft source only — bad grammar costs the speedup, never the request)+ opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8)+ self.assertEqual(opts[3], "not a grammar ::=")class ProtocolTest(unittest.TestCase):diff --git a/docs/grammar-draft.md b/docs/grammar-draft.mdindex 707fe5c6..6140cbbb 100644--- a/docs/grammar-draft.md+++ b/docs/grammar-draft.md@@ -83,3 +83,27 @@ this implementation adds: forced spans as a **draft source verified in the targeown forward** (lossless even under a wrong grammar, composes with MTP/n-gram in oneunion batch), deployed where the win is denominated in expert I/O rather thanforward passes.++## Server usage: `response_format` (OpenAI API)++The gateway (`openai_server.py`) turns `response_format` into a **per-request**+grammar carried to the engine over the `SUBMIT` protocol (optional 7th header+field, `gbytes`; 6-field headers from older clients remain valid — the field is+additive and back-compatible in both directions):++```jsonc+{"response_format": {"type": "json_object"}} // generic JSON grammar+{"response_format": {"type": "json_schema",+ "json_schema": {"schema": { ... }}}} // compiled by schema_gbnf.h+{"response_format": {"type": "gbnf", "grammar": "root ::= ..."}} // raw GBNF (extension)+```++Semantics are identical to `GRAMMAR=`/`SCHEMA=`: a **draft source, never a+sampling constraint**. A schema outside the supported subset, or malformed GBNF,+costs the speedup — never the request, never the output. Drafting engages for+greedy requests (`temperature: 0`); sampled requests run undrafted. Compile+overhead is negligible: ~8 µs/request for a typical schema, ~18 µs at the+32-level nesting cap (measured, M3 Max). Grammar payloads are capped at 1 MiB.+As with MTP (#100), a drafted greedy run may differ from an undrafted one in+near-tie tokens (the verify forward has a different batch shape); each output+is a valid greedy stream of its own forward shapes.