Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

serve: say why the model did not load, as a kind the gateway can name

Léon Avic Simmons72ce90a99d83
diff --git a/c/load_fail.h b/c/load_fail.h
new file mode 100644
index 00000000..e4f6c922
--- /dev/null
+++ b/c/load_fail.h
@@ -0,0 +1,103 @@
+/* load_fail.h -- the one exit for a load-time failure, classified.
+ *
+ * An engine that cannot load its model says why on stderr and exits 1. The
+ * server, reading stdout for READY, sees only EOF and reports "colibri engine
+ * exited unexpectedly" for an out-of-memory host, an unreadable path, a
+ * truncated shard and an unsupported dtype alike. coli_load_fail() keeps the
+ * stderr line as it was and, in serve mode (SERVE set, so stdout is the
+ * handshake channel), writes one classified line on stdout before exiting:
+ *
+ * LOAD_FAIL kind=<nomem|io|format|unsupported> <detail>
+ *
+ * It is the last line the engine writes; READY never follows it.
+ * nomem the host refused memory: ENOMEM/EAGAIN from malloc, mmap, mlock
+ * io the file could not be reached or read: ENOENT, EACCES, EIO, ...
+ * format the file was read but is not what it claims: short, bad header
+ * unsupported well-formed, but a dtype or geometry this engine does not serve
+ *
+ * Standalone on purpose: st.h includes it, and st.h is compiled as C, C++
+ * (.mm, .cpp) and CUDA, so nothing from the serve codec can come along. */
+#ifndef COLI_LOAD_FAIL_H
+#define COLI_LOAD_FAIL_H
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef enum {
+ COLI_LOAD_FAIL_NOMEM = 0,
+ COLI_LOAD_FAIL_IO,
+ COLI_LOAD_FAIL_FORMAT,
+ COLI_LOAD_FAIL_UNSUPPORTED
+} ColiLoadFailKind;
+
+static inline const char *coli_load_fail_kind_string(ColiLoadFailKind kind)
+{
+ switch (kind) {
+ case COLI_LOAD_FAIL_NOMEM: return "nomem";
+ case COLI_LOAD_FAIL_IO: return "io";
+ case COLI_LOAD_FAIL_FORMAT: return "format";
+ case COLI_LOAD_FAIL_UNSUPPORTED: return "unsupported";
+ }
+ return "io";
+}
+
+/* errno -> kind, for the sites where the OS said why. The host refusing
+ * memory is ENOMEM (malloc, mmap, a read into pages it cannot back), EAGAIN
+ * (mlock past RLIMIT_MEMLOCK, mmap with MAP_LOCKED) or ENOBUFS; every other
+ * errno is the path or the file. */
+static inline ColiLoadFailKind coli_load_fail_kind_from_errno(int err)
+{
+ switch (err) {
+ case ENOMEM:
+ case EAGAIN:
+#ifdef ENOBUFS
+ case ENOBUFS:
+#endif
+ return COLI_LOAD_FAIL_NOMEM;
+ default:
+ return COLI_LOAD_FAIL_IO;
+ }
+}
+
+/* The classified line on the handshake channel. A newline inside the detail
+ * would end the line early, so it is written as a space. */
+static inline int coli_serve_write_load_fail(FILE *output, ColiLoadFailKind kind,
+ const char *detail)
+{
+ if (fprintf(output, "LOAD_FAIL kind=%s ", coli_load_fail_kind_string(kind)) < 0)
+ return 0;
+ for (const char *p = detail ? detail : ""; *p; p++)
+ if (fputc(*p == '\n' || *p == '\r' ? ' ' : *p, output) == EOF) return 0;
+ return fputc('\n', output) != EOF && fflush(output) == 0;
+}
+
+#if defined(__GNUC__) || defined(__clang__)
+#define COLI_LOAD_FAIL_NORETURN __attribute__((noreturn))
+#define COLI_LOAD_FAIL_PRINTF(fmt, args) __attribute__((format(printf, fmt, args)))
+#elif defined(_MSC_VER)
+#define COLI_LOAD_FAIL_NORETURN __declspec(noreturn)
+#define COLI_LOAD_FAIL_PRINTF(fmt, args)
+#else
+#define COLI_LOAD_FAIL_NORETURN
+#define COLI_LOAD_FAIL_PRINTF(fmt, args)
+#endif
+
+/* Says why on stderr, as every load-time exit always has; says the kind on
+ * stdout when SERVE is set; exits 1. Never returns. */
+static inline COLI_LOAD_FAIL_NORETURN COLI_LOAD_FAIL_PRINTF(2, 3)
+void coli_load_fail(ColiLoadFailKind kind, const char *format, ...)
+{
+ char detail[1024];
+ va_list args;
+ va_start(args, format);
+ vsnprintf(detail, sizeof(detail), format, args);
+ va_end(args);
+ fprintf(stderr, "%s\n", detail);
+ if (getenv("SERVE")) coli_serve_write_load_fail(stdout, kind, detail);
+ exit(1);
+}
+
+#endif
diff --git a/c/openai_server.py b/c/openai_server.py
index 15390b82..6184d9cb 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -4651,11 +4651,42 @@ def _engine_extension_args(engine_k):
return {"logprobs": engine_k, "gbytes_before_ext": True}
+class EngineLoadError(RuntimeError):
+ """The engine said why it could not load, and exited before READY.
+
+ `LOAD_FAIL kind=<kind> <detail>` is the last line such an engine writes on the
+ handshake channel (see docs/serve_protocol.md): `kind` is nomem, io, format or
+ unsupported, `detail` the text it also wrote to stderr. Raised in place of the
+ bare "engine exited unexpectedly" so the log names the cause: an out-of-memory
+ host is not a bad file."""
+
+ def __init__(self, kind, detail):
+ super().__init__(f"colibri engine failed to load (kind={kind}): {detail}")
+ self.kind = kind
+ self.detail = detail
+
+
+def parse_load_fail(data):
+ """(kind, detail) from the last LOAD_FAIL line among the bytes an engine wrote
+ before it should have said READY; None when it wrote no such line."""
+ found = None
+ for line in data.decode("utf-8", "replace").splitlines():
+ fields = line.strip().split(None, 2)
+ if len(fields) >= 2 and fields[0] == "LOAD_FAIL" and fields[1].startswith("kind="):
+ found = (fields[1][len("kind="):], fields[2] if len(fields) > 2 else "")
+ return found
+
+
def read_engine_turn(stream, sentinel, on_bytes, caps=None):
pending = b""
while True:
byte = stream.read(1)
if byte == b"":
+ # The sentinel-length tail was held back in case it began the
+ # sentinel; at EOF it is the engine's last words (a LOAD_FAIL line
+ # ends there), so hand it over before giving up.
+ if pending:
+ on_bytes(pending)
raise RuntimeError("colibri engine exited unexpectedly")
pending += byte
if pending.endswith(sentinel):
@@ -4990,7 +5021,20 @@ class Engine:
self.profile = collections.deque(maxlen=PROFILE_TURNS) # per-turn phase timings
self.profile_seq = 0
self.caps = {} # the engine's CAPS handshake line, key=value
- read_engine_turn(self.process.stdout, READY, lambda _: None, self.caps)
+ # What the engine wrote before READY, the last 4 KiB of it: an engine that
+ # exits instead of saying READY leaves its LOAD_FAIL line there.
+ boot = bytearray()
+
+ def keep(data):
+ boot.extend(data)
+ del boot[:-4096]
+ try:
+ read_engine_turn(self.process.stdout, READY, keep, self.caps)
+ except RuntimeError:
+ failure = parse_load_fail(bytes(boot))
+ if failure is not None:
+ raise EngineLoadError(*failure) from None
+ raise
# True/False when the engine said whether it loaded a vision tower; None when
# it said nothing (an engine that predates CAPS, or a family without a tower).
self.vision = {"1": True, "0": False}.get(self.caps.get("vision"))
@@ -7662,7 +7706,14 @@ def serve(model, host="127.0.0.1", port=8000, model_id=None, api_key=None,
f"{runtime.info['default_height']}, {runtime.info['default_steps']} steps",
file=sys.stderr)
else:
- runtime = Engine(engine,model,cap,max_tokens,env,kv_slots,family)
+ try:
+ runtime = Engine(engine,model,cap,max_tokens,env,kv_slots,family)
+ except EngineLoadError as error:
+ # The engine said why: one line naming the kind, not a traceback
+ # that ends in "exited unexpectedly".
+ print(f"[gateway] engine load failed: kind={error.kind} {error.detail}",
+ file=sys.stderr)
+ sys.exit(1)
server.engine = runtime
if family.modality != "image":
# Said once at start-up, so a checkpoint that declares a tower it does
diff --git a/c/serve_codec.h b/c/serve_codec.h
index 35444da5..0b91c94d 100644
--- a/c/serve_codec.h
+++ b/c/serve_codec.h
@@ -10,6 +10,7 @@
#include <string.h>
#include "compat.h"
+#include "load_fail.h" /* LOAD_FAIL kind=<kind> <detail>: the handshake that never reaches READY */
/* Transport only: parse and emit frames. Admission, queueing, cancellation,
* KV ownership, scheduling, and generation remain with each family engine. */
diff --git a/c/st.h b/c/st.h
index 5d914550..bc6aaeaf 100644
--- a/c/st.h
+++ b/c/st.h
@@ -21,6 +21,7 @@
#include <sys/stat.h>
#include "json.h"
#include "compat.h"
+#include "load_fail.h"
/* tetto sulla dimensione dell'header safetensors: gli header reali sono piccoli
* (KB..pochi MB). Un file crafted che dichiara un hlen enorme causerebbe una
@@ -101,7 +102,7 @@ static int st_dtype_code(const char *s) {
* st_read_raw after validating the packed geometry). Same rule as the fp8
* codes above: a new number, read raw, refused by the float readers. */
if (!strcmp(s, "I32") || !strcmp(s, "U32")) return 7;
- fprintf(stderr, "unsupported dtype: %s\n", s); exit(1);
+ coli_load_fail(COLI_LOAD_FAIL_UNSUPPORTED, "unsupported dtype: %s", s);
}
/* Byte per elemento. UNICO posto che lo sa: prima la formula era ripetuta in tre
@@ -263,9 +264,10 @@ static void f16_to_f32_bulk(const uint16_t *src, float *dst, int64_t n) {
static int st_open_fd(shards *S, const char *path) {
for (int i = 0; i < S->nfd; i++) if (!strcmp(S->paths[i], path)) return S->fds[i];
int fd = open(path, COMPAT_O_RDONLY);
- if (fd < 0) { perror(path); exit(1); }
+ if (fd < 0) coli_load_fail(coli_load_fail_kind_from_errno(errno), "%s: %s", path, strerror(errno));
struct stat sb;
- if (fstat(fd, &sb) != 0) { perror("fstat shard"); close(fd); exit(1); }
+ if (fstat(fd, &sb) != 0) { int saved = errno; close(fd);
+ coli_load_fail(coli_load_fail_kind_from_errno(saved), "fstat shard: %s", strerror(saved)); }
S->paths[S->nfd] = strdup(path); S->fds[S->nfd] = fd;
S->sizes[S->nfd] = (int64_t)sb.st_size;
#ifdef O_DIRECT
@@ -389,14 +391,15 @@ static void st_pread_full(int fd, void *buf, int64_t n, int64_t off, const char
ssize_t r = pread(fd, p + got, (size_t)want, off + got);
if (r < 0) {
if (errno == EINTR) continue;
- fprintf(stderr, "%s: %s (off %lld, %lld/%lld bytes)\n", tag, strerror(errno),
- (long long)off, (long long)got, (long long)n);
- exit(1);
+ int saved = errno;
+ coli_load_fail(coli_load_fail_kind_from_errno(saved),
+ "%s: %s (off %lld, %lld/%lld bytes)", tag, strerror(saved),
+ (long long)off, (long long)got, (long long)n);
}
if (r == 0) {
- fprintf(stderr, "%s: short read at EOF (off %lld, %lld/%lld bytes) — truncated file?\n",
- tag, (long long)off, (long long)got, (long long)n);
- exit(1);
+ coli_load_fail(COLI_LOAD_FAIL_FORMAT,
+ "%s: short read at EOF (off %lld, %lld/%lld bytes) — truncated file?",
+ tag, (long long)off, (long long)got, (long long)n);
}
got += r;
}
@@ -588,7 +591,7 @@ static void st_index_load(st_index *ix, const char *dir) {
ix->map = map;
ix->hcap = 1; while (ix->hcap < map->len * 2) ix->hcap <<= 1;
ix->h = malloc((size_t)ix->hcap * sizeof(int));
- if (!ix->h) { fprintf(stderr, "OOM indexing model.safetensors.index.json\n"); exit(1); }
+ if (!ix->h) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "OOM indexing model.safetensors.index.json");
for (int i = 0; i < ix->hcap; i++) ix->h[i] = -1;
for (int i = 0; i < map->len; i++) {
uint64_t hh = st_hash(map->keys[i]) & (ix->hcap - 1);
@@ -620,7 +623,7 @@ static void st_index_free(st_index *ix) {
* returns how many shards this dir contributed. */
static void st_scan_dir(const char *dir, char files[][1024], int *nf, int *added) {
DIR *d = opendir(dir); struct dirent *e;
- if (!d) { perror(dir); exit(1); }
+ if (!d) coli_load_fail(coli_load_fail_kind_from_errno(errno), "%s: %s", dir, strerror(errno));
int base_n = *nf;
while ((e = readdir(d))) {
const char *dot = strrchr(e->d_name, '.');
@@ -713,17 +716,17 @@ static void st_init_multi(shards *S, const char *snap_dir, const char *extra_dir
* overflow (malloc(0) e poi hdr[hlen]=0 fuori limiti) o forzare una
* malloc gigante. */
if (fsz < 8 || hlen > (uint64_t)(fsz - 8) || hlen > (uint64_t)ST_MAX_HEADER) {
- fprintf(stderr, "%s: bad safetensors header length %llu (file %lld bytes)\n",
- files[fi], (unsigned long long)hlen, (long long)fsz); exit(1); }
+ coli_load_fail(COLI_LOAD_FAIL_FORMAT, "%s: bad safetensors header length %llu (file %lld bytes)",
+ files[fi], (unsigned long long)hlen, (long long)fsz); }
char *hdr = malloc(hlen + 1);
- if (!hdr) { perror("malloc safetensors header"); exit(1); }
+ if (!hdr) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "malloc safetensors header: %s", strerror(errno));
st_pread_full(fd, hdr, (int64_t)hlen, 8, "pread hdr");
hdr[hlen] = 0;
int64_t data_start = 8 + (int64_t)hlen;
char *arena = NULL;
jval *root = json_parse(hdr, &arena);
if (!root || root->t != J_OBJ) {
- fprintf(stderr, "%s: safetensors header is not a JSON object\n", files[fi]); exit(1); }
+ coli_load_fail(COLI_LOAD_FAIL_FORMAT, "%s: safetensors header is not a JSON object", files[fi]); }
st_fmt_stamp_ingest(S, root, files[fi]);
for (int i = 0; i < root->len; i++) {
const char *name = root->keys[i];
@@ -1073,7 +1076,7 @@ static int64_t st_read_f32(shards *S, const char *name, float *out, int drop) {
fprintf(stderr, "%s: tensor '%s' shape/bytes mismatch (numel %lld, %lld bytes, dtype %d) — refusing (hostile or corrupt file)\n",
name, name, (long long)t->numel, (long long)t->nbytes, t->dtype); exit(1); }
void *raw = malloc(t->nbytes);
- if (!raw) { fprintf(stderr, "malloc %lld bytes for tensor %s failed\n", (long long)t->nbytes, name); exit(1); }
+ if (!raw) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "malloc %lld bytes for tensor %s failed", (long long)t->nbytes, name);
st_pread_full(t->fd, raw, t->nbytes, t->off, "pread data");
if (t->dtype == 2) {
memcpy(out, raw, t->nbytes);
@@ -1158,7 +1161,7 @@ static int64_t st_read_scale_f32(shards *S, const char *name, float *out, int64_
fprintf(stderr, "scale %s: F8_E8M0 numel %lld disagrees with %lld bytes\n",
name, (long long)t->numel, (long long)t->nbytes); exit(1); }
uint8_t *raw = (uint8_t*)malloc((size_t)t->nbytes);
- if (!raw) { fprintf(stderr, "malloc %lld bytes for scale %s failed\n", (long long)t->nbytes, name); exit(1); }
+ if (!raw) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "malloc %lld bytes for scale %s failed", (long long)t->nbytes, name);
st_pread_full(t->fd, raw, t->nbytes, t->off, "pread ue8m0 scale");
for (int64_t i = 0; i < t->numel; i++) out[i] = ue8m0_to_f32(raw[i]);
free(raw);
@@ -1462,7 +1465,7 @@ static void st_read_slice_f32(shards *S, const char *name, int64_t elem_off, int
fprintf(stderr, "slice %s byte arithmetic/destination is invalid — refusing\n", name); exit(1); }
int64_t boff = t->off + elem_off * esz, nb = n_elems * esz;
void *raw = nb ? malloc((size_t)nb) : NULL;
- if (nb && !raw) { fprintf(stderr, "malloc %lld bytes for slice %s failed\n", (long long)nb, name); exit(1); }
+ if (nb && !raw) coli_load_fail(COLI_LOAD_FAIL_NOMEM, "malloc %lld bytes for slice %s failed", (long long)nb, name);
if (nb) st_pread_full(t->fd, raw, nb, boff, "pread slice"); /* dev #331: chunked + EINTR + honest short-read */
if (nb) {
if (t->dtype == 2) memcpy(out, raw, (size_t)nb);
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 809d0371..ae552c2a 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -7413,6 +7413,63 @@ class EngineCapsTest(unittest.TestCase):
engine.close()
+class EngineLoadFailTest(unittest.TestCase):
+ """An engine that cannot load says `LOAD_FAIL kind=<kind> <detail>` and exits
+ before READY; the gateway raises the kind, not "exited unexpectedly"."""
+
+ def _engine(self, stdout_bytes):
+ process = FakeProcess(lambda _process, _frame: None)
+ process.stdout = BlockingStream(stdout_bytes)
+ process.stdout.close() # the engine is gone: EOF right after what it said
+ with patch("openai_server.ARCH", "glm53"), \
+ patch("openai_server.subprocess.Popen", return_value=process):
+ return Engine("glm53", "model")
+
+ def test_load_fail_line_is_raised_as_its_kind(self):
+ detail = "model-00007.safetensors: short read at EOF (off 8, 0/16 bytes)"
+ for kind in ("nomem", "io", "format", "unsupported"):
+ with self.subTest(kind=kind):
+ with self.assertRaises(openai_server.EngineLoadError) as caught:
+ self._engine(f"LOAD_FAIL kind={kind} {detail}\n".encode())
+ self.assertEqual(caught.exception.kind, kind)
+ self.assertEqual(caught.exception.detail, detail)
+ self.assertIn(f"kind={kind}", str(caught.exception))
+ self.assertNotIn("unexpectedly", str(caught.exception))
+
+ def test_engine_that_says_nothing_is_still_exited_unexpectedly(self):
+ with self.assertRaisesRegex(RuntimeError, "exited unexpectedly") as caught:
+ self._engine(b"")
+ self.assertNotIsInstance(caught.exception, openai_server.EngineLoadError)
+
+ def test_parse_load_fail_takes_the_last_line_and_keeps_the_detail_whole(self):
+ parse = openai_server.parse_load_fail
+ self.assertIsNone(parse(b"HWINFO 8 64 32 0 0 cpu|none\n"))
+ self.assertEqual(parse(b"noise\nLOAD_FAIL kind=nomem malloc 12 bytes for tensor a.b failed\n"),
+ ("nomem", "malloc 12 bytes for tensor a.b failed"))
+ self.assertEqual(parse(b"LOAD_FAIL kind=io\n"), ("io", ""))
+
+ def test_serve_logs_the_kind_and_exits(self):
+ error = openai_server.EngineLoadError("nomem", "malloc 12 bytes for tensor a.b failed")
+ stderr = io.StringIO()
+ # serve() sets the module's ARCH / CHAT_FLAVOR for the family it resolves;
+ # patched here so the rest of the suite keeps its defaults.
+ with patch("openai_server.ARCH", openai_server.ARCH), \
+ patch("openai_server.CHAT_FLAVOR", openai_server.CHAT_FLAVOR), \
+ patch("openai_server.Engine", side_effect=error), \
+ patch("openai_server.resolve_model") as resolve, \
+ patch("openai_server.default_engine", return_value="colibri"), \
+ patch("openai_server.detect_chat_flavor", return_value=None), \
+ patch("openai_server.APIServer") as api_server, \
+ patch("sys.stderr", stderr):
+ resolve.return_value.descriptor = openai_server.family_by_id("glm53")
+ with self.assertRaises(SystemExit) as caught:
+ openai_server.serve("model", "127.0.0.1", 8000, "id", None)
+ self.assertEqual(caught.exception.code, 1)
+ self.assertIn("[gateway] engine load failed: kind=nomem malloc 12 bytes for tensor a.b failed",
+ stderr.getvalue())
+ api_server.return_value.server_close.assert_called_once()
+
+
class ServedModalityTest(unittest.TestCase):
"""What /v1/models says it accepts is what the engine loaded. The tower is a
property of the checkpoint (a glm53 export can carry vision_config and no
diff --git a/c/tests/test_serve_codec.c b/c/tests/test_serve_codec.c
index 51d81e06..1e502f84 100644
--- a/c/tests/test_serve_codec.c
+++ b/c/tests/test_serve_codec.c
@@ -1,4 +1,5 @@
#include <assert.h>
+#include <errno.h>
#include <stdio.h>
#include <string.h>
@@ -234,6 +235,47 @@ static void test_allocation_failures_are_fatal_to_the_caller(void)
}
}
+/* The load-time failure class: errno maps to nomem or io, the kinds have
+ * their wire words, and the line is one line (a newline in the detail is a
+ * space), so the server can read the kind and keep the detail whole. */
+static void test_load_fail_kinds_and_line(void)
+{
+ assert(coli_load_fail_kind_from_errno(ENOMEM) == COLI_LOAD_FAIL_NOMEM);
+ assert(coli_load_fail_kind_from_errno(EAGAIN) == COLI_LOAD_FAIL_NOMEM);
+ assert(coli_load_fail_kind_from_errno(ENOENT) == COLI_LOAD_FAIL_IO);
+ assert(coli_load_fail_kind_from_errno(EACCES) == COLI_LOAD_FAIL_IO);
+ assert(coli_load_fail_kind_from_errno(EIO) == COLI_LOAD_FAIL_IO);
+ assert(coli_load_fail_kind_from_errno(EMFILE) == COLI_LOAD_FAIL_IO);
+ assert(coli_load_fail_kind_from_errno(0) == COLI_LOAD_FAIL_IO);
+ assert(strcmp(coli_load_fail_kind_string(COLI_LOAD_FAIL_NOMEM), "nomem") == 0);
+ assert(strcmp(coli_load_fail_kind_string(COLI_LOAD_FAIL_IO), "io") == 0);
+ assert(strcmp(coli_load_fail_kind_string(COLI_LOAD_FAIL_FORMAT), "format") == 0);
+ assert(strcmp(coli_load_fail_kind_string(COLI_LOAD_FAIL_UNSUPPORTED), "unsupported") == 0);
+
+ FILE *output = tmpfile();
+ assert(output);
+ binary_stream(output);
+ assert(coli_serve_write_load_fail(
+ output, COLI_LOAD_FAIL_FORMAT,
+ "model-00007.safetensors: short read at EOF (off 8, 0/16 bytes)\ntruncated?"));
+ static const char expected[] =
+ "LOAD_FAIL kind=format model-00007.safetensors: short read at EOF "
+ "(off 8, 0/16 bytes) truncated?\n";
+ unsigned char bytes[256];
+ size_t count = read_output(output, bytes, sizeof(bytes));
+ assert(count == sizeof(expected) - 1);
+ assert(memcmp(bytes, expected, count) == 0);
+ fclose(output);
+
+ output = tmpfile();
+ assert(output);
+ binary_stream(output);
+ assert(coli_serve_write_load_fail(output, COLI_LOAD_FAIL_NOMEM, NULL));
+ count = read_output(output, bytes, sizeof(bytes));
+ assert(count == 22 && memcmp(bytes, "LOAD_FAIL kind=nomem \n", 22) == 0);
+ fclose(output);
+}
+
int main(void)
{
test_submit_and_controls();
@@ -241,6 +283,7 @@ int main(void)
test_writer_golden_bytes();
test_ready_caps_golden_bytes();
test_allocation_failures_are_fatal_to_the_caller();
+ test_load_fail_kinds_and_line();
puts("serve codec tests: ok");
return 0;
}
diff --git a/docs/serve_protocol.md b/docs/serve_protocol.md
index bbd0b0a4..11704a8c 100644
--- a/docs/serve_protocol.md
+++ b/docs/serve_protocol.md
@@ -38,6 +38,25 @@ declares a tower whose tensors are not in the container). Engines that say nothi
leave the server's flag unknown. glm53, qwen38, qwen36 and deepseek_v41 emit it;
`openai_server.py` derives `/v1/models` `input_modalities` from it.
+An engine that cannot load its model says why on this channel, instead of
+`READY`, and exits:
+
+```
+LOAD_FAIL kind=<nomem|io|format|unsupported> <detail>
+```
+
+It is the last line such an engine writes. `nomem`: the host refused memory
+(ENOMEM/EAGAIN from malloc, mmap or mlock); `io`: the file could not be reached or
+read (ENOENT, EACCES, EIO, ...); `format`: the file was read but is not what it
+claims (short read, bad header); `unsupported`: well-formed, but a dtype or geometry
+this engine does not serve. The detail is the same text the engine wrote to
+stderr. `openai_server.py` raises `EngineLoadError(kind, detail)` from it and logs
+`[gateway] engine load failed: kind=<kind> <detail>`; an engine that exits with no
+such line is still reported as "colibri engine exited unexpectedly". The shared
+safetensors path (`st.h`, every family) emits it through `coli_load_fail()`
+(`load_fail.h`); a family's own load-time refusals adopt the same call as they are
+touched.
+
## Requests (server → engine)
```
@@ -108,6 +127,7 @@ turn: `HWINFO`, `PERF`, `ENTROPY`, `GPUS`, `TIERS`, `EMAP`, `HITS` (formats belo
| `TIERS` | `TIERS <vram> <ram> <disk> <vram_gb> <ram_gb>` | expert count per tier + resident bytes |
| `HWINFO` | `HWINFO <cores> <ram_total> <ram_avail> <ngpu> <vram_total> <cpu>\|<gpu>` | host snapshot (GBs are floats) |
| `CAPS` | `CAPS key=value ...` | handshake only, between `READY` and `STAT`: what the engine loaded (`vision=0\|1`) |
+| `LOAD_FAIL` | `LOAD_FAIL kind=<kind> <detail>` | handshake only, instead of `READY`: why the model did not load (`nomem`, `io`, `format`, `unsupported`); the engine exits after it |
| `EMAP` | `EMAP <rows> <cols> <hex>` | one byte per expert, row-major over `rows×cols` (sparse layers +MTP × experts): `byte = (tier<<6) \| heat` — 2-bit tier (0 disk / 1 RAM / 2 VRAM), 6-bit log₂-bucketed usage heat |
| `HITS` | `HITS <rows> <cols> <hex>` | 1 bit per expert, experts routed since the previous `HITS` |
| `PERF` | `PERF <id> <dt> <t_edisk> <t_ewait> <t_emm> <t_attn> <t_kvb> <t_head>` | this turn's PROFILO deltas, seconds |