Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(segment): add engine-neutral runtime ABI

JustVuggd9c94fa9517d
diff --git a/README.md b/README.md
index b4effc89..0328581f 100644
--- a/README.md
+++ b/README.md
@@ -507,6 +507,7 @@ Two things that differ per model, both documented in the per-model page:
| Vulkan backend (any GPU: AMD via RADV, incl. cards ROCm dropped) | [docs/vulkan.md](docs/vulkan.md) |
| Apple Silicon Metal backend | [docs/metal.md](docs/metal.md) |
| OpenAI-compatible API, KV slots, web dashboard | [docs/api.md](docs/api.md) |
+| Experimental layer-segment embedding ABI | [docs/segment-runtime.md](docs/segment-runtime.md) |
| Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) |
| Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
diff --git a/c/Makefile b/c/Makefile
index 798ce387..2769c0d6 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1129,6 +1129,9 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
tests/test_serve_codec$(EXE): tests/test_serve_codec.c serve_codec.h compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_segment_runtime$(EXE): tests/test_segment_runtime.c segment_runtime.c segment_runtime.h
+ $(CC) $(CFLAGS) tests/test_segment_runtime.c segment_runtime.c -o $@ $(LDFLAGS)
+
tests/test_inkling_serve_framing$(EXE): tests/test_inkling_serve_framing.c inkling.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h $(INK_CUDA_OBJ) $(METAL_OBJ)
$(CC) $(CFLAGS) $< $(INK_CUDA_OBJ) $(METAL_OBJ) -o $@ $(LDFLAGS)
diff --git a/c/segment_runtime.c b/c/segment_runtime.c
new file mode 100644
index 00000000..b5f6c066
--- /dev/null
+++ b/c/segment_runtime.c
@@ -0,0 +1,257 @@
+#include "segment_runtime.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define COLI_SEGMENT_MAX_ADAPTERS 16
+
+struct ColiSegmentEngine {
+ const ColiSegmentAdapter *adapter;
+ void *impl;
+ ColiSegmentCapabilities capabilities;
+ uint32_t context_tokens;
+ unsigned active_sessions;
+};
+
+struct ColiSegmentSession {
+ ColiSegmentEngine *engine;
+ void *impl;
+ uint32_t context_tokens;
+};
+
+static const ColiSegmentAdapter *g_adapters[COLI_SEGMENT_MAX_ADAPTERS];
+static int g_adapter_count;
+
+static int set_error(char *error, size_t error_size, const char *message) {
+ if (error && error_size) snprintf(error, error_size, "%s", message);
+ return -1;
+}
+
+static int id_valid(const char *id) {
+ if (!id || !*id) return 0;
+ size_t length = strlen(id);
+ if (length >= COLI_SEGMENT_ENGINE_ID_CAP) return 0;
+ for (size_t i = 0; i < length; i++) {
+ unsigned char c = (unsigned char)id[i];
+ if (!((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') ||
+ c == '_' || c == '-'))
+ return 0;
+ }
+ return 1;
+}
+
+static int fixed_string_valid(const char *value, size_t capacity) {
+ return value && value[0] && memchr(value, '\0', capacity) != NULL;
+}
+
+static size_t dtype_size(uint32_t dtype) {
+ switch (dtype) {
+ case COLI_SEGMENT_DTYPE_F32: return 4;
+ case COLI_SEGMENT_DTYPE_F16:
+ case COLI_SEGMENT_DTYPE_BF16: return 2;
+ default: return 0;
+ }
+}
+
+static int capabilities_valid(const ColiSegmentCapabilities *capabilities,
+ const ColiSegmentAdapter *adapter) {
+ int has_snapshot = adapter->session_snapshot && adapter->session_restore;
+ return capabilities->struct_size == sizeof(*capabilities) &&
+ capabilities->abi_version == COLI_SEGMENT_ABI_VERSION &&
+ fixed_string_valid(capabilities->engine_id,
+ sizeof(capabilities->engine_id)) &&
+ strcmp(capabilities->engine_id, adapter->engine_id) == 0 &&
+ fixed_string_valid(capabilities->state_schema,
+ sizeof(capabilities->state_schema)) &&
+ fixed_string_valid(capabilities->numeric_class,
+ sizeof(capabilities->numeric_class)) &&
+ dtype_size(capabilities->state_dtype) != 0 &&
+ capabilities->state_width && capabilities->num_layers &&
+ !!(capabilities->flags & COLI_SEGMENT_CAP_SNAPSHOT) == has_snapshot;
+}
+
+int coli_segment_adapter_register(const ColiSegmentAdapter *adapter) {
+ if (!adapter || adapter->struct_size < sizeof(*adapter) ||
+ adapter->abi_version != COLI_SEGMENT_ABI_VERSION ||
+ !id_valid(adapter->engine_id) || !adapter->engine_open ||
+ !adapter->engine_destroy ||
+ !adapter->session_create || !adapter->session_destroy ||
+ !adapter->session_run ||
+ !!adapter->session_snapshot != !!adapter->session_restore)
+ return -1;
+ for (int i = 0; i < g_adapter_count; i++)
+ if (strcmp(g_adapters[i]->engine_id, adapter->engine_id) == 0)
+ return -1;
+ if (g_adapter_count >= COLI_SEGMENT_MAX_ADAPTERS) return -1;
+ g_adapters[g_adapter_count++] = adapter;
+ return 0;
+}
+
+const ColiSegmentAdapter *coli_segment_adapter_lookup(const char *engine_id) {
+ if (!engine_id) return NULL;
+ for (int i = 0; i < g_adapter_count; i++)
+ if (strcmp(g_adapters[i]->engine_id, engine_id) == 0)
+ return g_adapters[i];
+ return NULL;
+}
+
+int coli_segment_adapter_count(void) { return g_adapter_count; }
+
+int coli_segment_engine_open(const char *engine_id,
+ const ColiSegmentEngineOptions *options,
+ ColiSegmentEngine **engine,
+ char *error, size_t error_size) {
+ if (!engine) return set_error(error, error_size, "missing segment engine output");
+ *engine = NULL;
+ if (!options || options->struct_size < sizeof(*options) ||
+ !options->model_dir || !*options->model_dir ||
+ options->layer_begin >= options->layer_end || !options->context_tokens)
+ return set_error(error, error_size, "invalid segment engine options");
+ const ColiSegmentAdapter *adapter = coli_segment_adapter_lookup(engine_id);
+ if (!adapter)
+ return set_error(error, error_size, "segment engine is not registered");
+
+ ColiSegmentEngine *created = (ColiSegmentEngine *)calloc(1, sizeof(*created));
+ if (!created) return set_error(error, error_size, "out of memory opening segment engine");
+ created->adapter = adapter;
+ created->capabilities.struct_size = sizeof(created->capabilities);
+ created->capabilities.abi_version = COLI_SEGMENT_ABI_VERSION;
+ if (adapter->engine_open(&created->impl, &created->capabilities, options,
+ error, error_size) ||
+ !created->impl) {
+ if (created->impl) adapter->engine_destroy(created->impl);
+ free(created);
+ return -1;
+ }
+ if (!capabilities_valid(&created->capabilities, adapter) ||
+ options->layer_end > created->capabilities.num_layers ||
+ (created->capabilities.max_context_tokens &&
+ options->context_tokens > created->capabilities.max_context_tokens)) {
+ adapter->engine_destroy(created->impl);
+ free(created);
+ return set_error(error, error_size,
+ "segment adapter returned incompatible capabilities");
+ }
+ created->context_tokens = options->context_tokens;
+ *engine = created;
+ return 0;
+}
+
+int coli_segment_engine_capabilities(const ColiSegmentEngine *engine,
+ ColiSegmentCapabilities *capabilities,
+ char *error, size_t error_size) {
+ if (!engine || !capabilities ||
+ capabilities->struct_size < sizeof(*capabilities))
+ return set_error(error, error_size,
+ "invalid segment capabilities buffer");
+ memcpy(capabilities, &engine->capabilities, sizeof(*capabilities));
+ return 0;
+}
+
+int coli_segment_engine_close(ColiSegmentEngine *engine,
+ char *error, size_t error_size) {
+ if (!engine) return 0;
+ if (engine->active_sessions)
+ return set_error(error, error_size,
+ "cannot close segment engine with live sessions");
+ engine->adapter->engine_destroy(engine->impl);
+ free(engine);
+ return 0;
+}
+
+int coli_segment_session_create(ColiSegmentEngine *engine,
+ const ColiSegmentSessionOptions *options,
+ ColiSegmentSession **session,
+ char *error, size_t error_size) {
+ if (!session) return set_error(error, error_size, "missing segment session output");
+ *session = NULL;
+ if (!engine || !options || options->struct_size < sizeof(*options) ||
+ !options->context_tokens ||
+ options->context_tokens > engine->context_tokens)
+ return set_error(error, error_size, "invalid segment session options");
+ ColiSegmentSession *created =
+ (ColiSegmentSession *)calloc(1, sizeof(*created));
+ if (!created)
+ return set_error(error, error_size, "out of memory creating segment session");
+ created->engine = engine;
+ if (engine->adapter->session_create(engine->impl, &created->impl, options,
+ error, error_size) || !created->impl) {
+ if (created->impl) engine->adapter->session_destroy(created->impl);
+ free(created);
+ return -1;
+ }
+ created->context_tokens = options->context_tokens;
+ engine->active_sessions++;
+ *session = created;
+ return 0;
+}
+
+void coli_segment_session_destroy(ColiSegmentSession *session) {
+ if (!session) return;
+ ColiSegmentEngine *engine = session->engine;
+ engine->adapter->session_destroy(session->impl);
+ if (engine->active_sessions) engine->active_sessions--;
+ free(session);
+}
+
+int coli_segment_run(ColiSegmentSession *session,
+ const ColiSegmentRunRequest *request,
+ char *error, size_t error_size) {
+ if (!session || !request || request->struct_size < sizeof(*request) ||
+ !request->rows || !request->input || !request->output ||
+ !request->input_bytes || !request->output_bytes)
+ return set_error(error, error_size, "invalid segment run request");
+
+ const ColiSegmentCapabilities *capabilities = &session->engine->capabilities;
+ if (capabilities->max_batch_rows && request->rows > capabilities->max_batch_rows)
+ return set_error(error, error_size, "segment batch exceeds capabilities");
+ if (!!request->token_ids != !!request->token_count)
+ return set_error(error, error_size,
+ "segment token IDs and count must both be present");
+ if ((capabilities->flags & COLI_SEGMENT_CAP_TOKEN_IDS) &&
+ request->token_count != request->rows)
+ return set_error(error, error_size, "segment engine requires one token ID per row");
+ if (request->token_count && request->token_count != request->rows)
+ return set_error(error, error_size, "segment token count does not match rows");
+ if (request->position > session->context_tokens ||
+ request->rows > session->context_tokens - request->position)
+ return set_error(error, error_size, "segment run exceeds session context");
+ size_t element_size = dtype_size(capabilities->state_dtype);
+ if (capabilities->state_width > SIZE_MAX / request->rows ||
+ (size_t)request->rows * capabilities->state_width >
+ SIZE_MAX / element_size)
+ return set_error(error, error_size, "segment activation size overflows");
+ size_t expected_bytes =
+ (size_t)request->rows * capabilities->state_width * element_size;
+ if (request->input_bytes != expected_bytes ||
+ request->output_bytes != expected_bytes)
+ return set_error(error, error_size,
+ "segment activation buffer has the wrong size");
+ if (request->should_cancel && request->should_cancel(request->cancel_user_data))
+ return set_error(error, error_size, "segment run cancelled");
+ return session->engine->adapter->session_run(session->impl, request,
+ error, error_size);
+}
+
+int coli_segment_snapshot(ColiSegmentSession *session,
+ ColiSegmentWriteFn write_fn, void *write_user_data,
+ char *error, size_t error_size) {
+ if (!session || !write_fn)
+ return set_error(error, error_size, "invalid segment snapshot request");
+ if (!session->engine->adapter->session_snapshot)
+ return set_error(error, error_size, "segment snapshots are not supported");
+ return session->engine->adapter->session_snapshot(
+ session->impl, write_fn, write_user_data, error, error_size);
+}
+
+int coli_segment_restore(ColiSegmentSession *session,
+ ColiSegmentReadFn read_fn, void *read_user_data,
+ char *error, size_t error_size) {
+ if (!session || !read_fn)
+ return set_error(error, error_size, "invalid segment restore request");
+ if (!session->engine->adapter->session_restore)
+ return set_error(error, error_size, "segment restore is not supported");
+ return session->engine->adapter->session_restore(
+ session->impl, read_fn, read_user_data, error, error_size);
+}
diff --git a/c/segment_runtime.h b/c/segment_runtime.h
new file mode 100644
index 00000000..4b970d10
--- /dev/null
+++ b/c/segment_runtime.h
@@ -0,0 +1,189 @@
+#ifndef COLIBRI_SEGMENT_RUNTIME_H
+#define COLIBRI_SEGMENT_RUNTIME_H
+
+/*
+ * Public, engine-neutral layer-segment runtime.
+ *
+ * Colibri owns model math, weights, accelerators and sequence state. A
+ * distributed caller owns transport, placement, leases and network session
+ * identifiers. This API is the boundary between the two: adapters keep all
+ * engine-specific objects opaque while callers can open a layer range, create
+ * isolated sessions, run activations through it and stream snapshots.
+ *
+ * Version 1 is additive and unused by the existing CLI/serve paths. Adapters
+ * register before the first lookup (normally from a link-time constructor),
+ * after which the registry is read-only and safe for concurrent lookup.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define COLI_SEGMENT_ABI_VERSION 1u
+#define COLI_SEGMENT_ENGINE_ID_CAP 64u
+#define COLI_SEGMENT_STATE_SCHEMA_CAP 128u
+#define COLI_SEGMENT_NUMERIC_CLASS_CAP 96u
+
+typedef struct ColiSegmentEngine ColiSegmentEngine;
+typedef struct ColiSegmentSession ColiSegmentSession;
+
+typedef enum {
+ COLI_SEGMENT_DTYPE_INVALID = 0,
+ COLI_SEGMENT_DTYPE_F32 = 1,
+ COLI_SEGMENT_DTYPE_F16 = 2,
+ COLI_SEGMENT_DTYPE_BF16 = 3,
+} ColiSegmentDType;
+
+enum {
+ COLI_SEGMENT_CAP_TOKEN_IDS = UINT64_C(1) << 0,
+ COLI_SEGMENT_CAP_SNAPSHOT = UINT64_C(1) << 1,
+ COLI_SEGMENT_CAP_RANGE_NATIVE = UINT64_C(1) << 2,
+ COLI_SEGMENT_CAP_MULTI_SESSION = UINT64_C(1) << 3,
+ COLI_SEGMENT_CAP_CPU = UINT64_C(1) << 8,
+ COLI_SEGMENT_CAP_CUDA = UINT64_C(1) << 9,
+ COLI_SEGMENT_CAP_HIP = UINT64_C(1) << 10,
+ COLI_SEGMENT_CAP_METAL = UINT64_C(1) << 11,
+ COLI_SEGMENT_CAP_VULKAN = UINT64_C(1) << 12,
+};
+
+#define COLI_SEGMENT_CAP_BACKEND_MASK \
+ (COLI_SEGMENT_CAP_CPU | COLI_SEGMENT_CAP_CUDA | COLI_SEGMENT_CAP_HIP | \
+ COLI_SEGMENT_CAP_METAL | COLI_SEGMENT_CAP_VULKAN)
+
+/* Filled by the adapter during engine_open, after it has inspected the model.
+ * Fixed-size identity fields make the record safe to copy across a local ABI
+ * boundary. state_width is the activation width on the segment wire; it need
+ * not equal the model hidden size (DeepSeek V4 mHC is one example). */
+typedef struct {
+ uint32_t struct_size;
+ uint32_t abi_version;
+ uint64_t flags;
+ char engine_id[COLI_SEGMENT_ENGINE_ID_CAP];
+ char state_schema[COLI_SEGMENT_STATE_SCHEMA_CAP];
+ char numeric_class[COLI_SEGMENT_NUMERIC_CLASS_CAP];
+ uint32_t state_dtype;
+ uint32_t state_width;
+ uint32_t max_batch_rows;
+ uint32_t max_context_tokens;
+ uint32_t num_layers;
+ uint32_t reserved_u32[7];
+ uint64_t reserved_u64[4];
+} ColiSegmentCapabilities;
+
+/* layer_begin is inclusive and layer_end is exclusive. backend_mask == 0
+ * lets the adapter select its normal local policy. memory_limit_bytes == 0
+ * uses the adapter's ordinary automatic budget. */
+typedef struct {
+ uint32_t struct_size;
+ const char *model_dir;
+ uint32_t layer_begin;
+ uint32_t layer_end;
+ uint32_t context_tokens;
+ uint32_t reserved_u32;
+ uint64_t memory_limit_bytes;
+ uint64_t backend_mask;
+ const void *resource_plan;
+ size_t resource_plan_size;
+} ColiSegmentEngineOptions;
+
+typedef struct {
+ uint32_t struct_size;
+ uint32_t context_tokens;
+ uint64_t memory_limit_bytes;
+ uint64_t reserved_u64[4];
+} ColiSegmentSessionOptions;
+
+typedef int (*ColiSegmentCancelFn)(void *user_data);
+
+/* input and output each hold rows * state_width values of the advertised
+ * dtype. token_ids is either NULL/0 or exactly rows entries. Engines whose
+ * capabilities include COLI_SEGMENT_CAP_TOKEN_IDS require the latter. */
+typedef struct {
+ uint32_t struct_size;
+ uint32_t rows;
+ uint64_t position;
+ const int32_t *token_ids;
+ size_t token_count;
+ const void *input;
+ size_t input_bytes;
+ void *output;
+ size_t output_bytes;
+ ColiSegmentCancelFn should_cancel;
+ void *cancel_user_data;
+ uint64_t reserved_u64[4];
+} ColiSegmentRunRequest;
+
+/* Snapshot formats are private to the adapter and named by state_schema.
+ * Streaming callbacks avoid a second full-state allocation in Colibri or in
+ * a network daemon. A callback returns zero on success, non-zero to abort. */
+typedef int (*ColiSegmentWriteFn)(void *user_data, const void *data, size_t size);
+typedef int (*ColiSegmentReadFn)(void *user_data, void *data, size_t size);
+
+typedef struct {
+ uint32_t struct_size;
+ uint32_t abi_version;
+ const char *engine_id;
+
+ int (*engine_open)(void **engine_impl,
+ ColiSegmentCapabilities *capabilities,
+ const ColiSegmentEngineOptions *options,
+ char *error, size_t error_size);
+ void (*engine_destroy)(void *engine_impl);
+ int (*session_create)(void *engine_impl, void **session_impl,
+ const ColiSegmentSessionOptions *options,
+ char *error, size_t error_size);
+ void (*session_destroy)(void *session_impl);
+ int (*session_run)(void *session_impl,
+ const ColiSegmentRunRequest *request,
+ char *error, size_t error_size);
+ int (*session_snapshot)(void *session_impl, ColiSegmentWriteFn write_fn,
+ void *write_user_data,
+ char *error, size_t error_size);
+ int (*session_restore)(void *session_impl, ColiSegmentReadFn read_fn,
+ void *read_user_data,
+ char *error, size_t error_size);
+
+ void (*reserved_fn[8])(void);
+} ColiSegmentAdapter;
+
+/* Registering is an initialization-time operation. Duplicate engine IDs and
+ * malformed/currently incompatible adapters are rejected. Complete all
+ * registration before lookups or execution begin. */
+int coli_segment_adapter_register(const ColiSegmentAdapter *adapter);
+const ColiSegmentAdapter *coli_segment_adapter_lookup(const char *engine_id);
+int coli_segment_adapter_count(void);
+
+int coli_segment_engine_open(const char *engine_id,
+ const ColiSegmentEngineOptions *options,
+ ColiSegmentEngine **engine,
+ char *error, size_t error_size);
+int coli_segment_engine_capabilities(const ColiSegmentEngine *engine,
+ ColiSegmentCapabilities *capabilities,
+ char *error, size_t error_size);
+/* Refuses to close an engine while sessions are alive. */
+int coli_segment_engine_close(ColiSegmentEngine *engine,
+ char *error, size_t error_size);
+
+int coli_segment_session_create(ColiSegmentEngine *engine,
+ const ColiSegmentSessionOptions *options,
+ ColiSegmentSession **session,
+ char *error, size_t error_size);
+void coli_segment_session_destroy(ColiSegmentSession *session);
+int coli_segment_run(ColiSegmentSession *session,
+ const ColiSegmentRunRequest *request,
+ char *error, size_t error_size);
+int coli_segment_snapshot(ColiSegmentSession *session,
+ ColiSegmentWriteFn write_fn, void *write_user_data,
+ char *error, size_t error_size);
+int coli_segment_restore(ColiSegmentSession *session,
+ ColiSegmentReadFn read_fn, void *read_user_data,
+ char *error, size_t error_size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* COLIBRI_SEGMENT_RUNTIME_H */
diff --git a/c/tests/test_segment_runtime.c b/c/tests/test_segment_runtime.c
new file mode 100644
index 00000000..961b0fde
--- /dev/null
+++ b/c/tests/test_segment_runtime.c
@@ -0,0 +1,251 @@
+#include "../segment_runtime.h"
+
+#include <assert.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+ uint32_t begin, end, context;
+} FakeEngine;
+
+typedef struct {
+ FakeEngine *engine;
+ uint64_t position;
+} FakeSession;
+
+typedef struct {
+ unsigned char bytes[64];
+ size_t length, offset;
+} MemoryStream;
+
+static void fake_capabilities(ColiSegmentCapabilities *capabilities) {
+ capabilities->abi_version = COLI_SEGMENT_ABI_VERSION;
+ capabilities->flags = COLI_SEGMENT_CAP_TOKEN_IDS |
+ COLI_SEGMENT_CAP_SNAPSHOT |
+ COLI_SEGMENT_CAP_RANGE_NATIVE |
+ COLI_SEGMENT_CAP_MULTI_SESSION |
+ COLI_SEGMENT_CAP_CPU;
+ snprintf(capabilities->engine_id, sizeof(capabilities->engine_id), "fake");
+ snprintf(capabilities->state_schema, sizeof(capabilities->state_schema),
+ "fake-state-v1");
+ snprintf(capabilities->numeric_class, sizeof(capabilities->numeric_class),
+ "fake-f32-strict-v1");
+ capabilities->state_dtype = COLI_SEGMENT_DTYPE_F32;
+ capabilities->state_width = 4;
+ capabilities->max_batch_rows = 8;
+ capabilities->max_context_tokens = 32;
+ capabilities->num_layers = 6;
+}
+
+static int fake_engine_open(void **engine_impl,
+ ColiSegmentCapabilities *capabilities,
+ const ColiSegmentEngineOptions *options,
+ char *error, size_t error_size) {
+ (void)error; (void)error_size;
+ FakeEngine *engine = (FakeEngine *)calloc(1, sizeof(*engine));
+ if (!engine) return -1;
+ engine->begin = options->layer_begin;
+ engine->end = options->layer_end;
+ engine->context = options->context_tokens;
+ fake_capabilities(capabilities);
+ *engine_impl = engine;
+ return 0;
+}
+
+static void fake_engine_destroy(void *engine_impl) { free(engine_impl); }
+
+static int fake_session_create(void *engine_impl, void **session_impl,
+ const ColiSegmentSessionOptions *options,
+ char *error, size_t error_size) {
+ FakeEngine *engine = (FakeEngine *)engine_impl;
+ if (options->context_tokens > engine->context) {
+ if (error && error_size) snprintf(error, error_size, "context too large");
+ return -1;
+ }
+ FakeSession *session = (FakeSession *)calloc(1, sizeof(*session));
+ if (!session) return -1;
+ session->engine = engine;
+ *session_impl = session;
+ return 0;
+}
+
+static void fake_session_destroy(void *session_impl) { free(session_impl); }
+
+static int fake_session_run(void *session_impl,
+ const ColiSegmentRunRequest *request,
+ char *error, size_t error_size) {
+ FakeSession *session = (FakeSession *)session_impl;
+ size_t count = (size_t)request->rows * 4;
+ if (request->position != session->position ||
+ request->input_bytes != count * sizeof(float) ||
+ request->output_bytes != count * sizeof(float)) {
+ if (error && error_size) snprintf(error, error_size, "bad fake run");
+ return -1;
+ }
+ const float *input = (const float *)request->input;
+ float *output = (float *)request->output;
+ float add = (float)(session->engine->end - session->engine->begin);
+ for (size_t i = 0; i < count; i++) output[i] = input[i] + add;
+ session->position += request->rows;
+ return 0;
+}
+
+static int memory_write(void *user_data, const void *data, size_t size) {
+ MemoryStream *stream = (MemoryStream *)user_data;
+ if (size > sizeof(stream->bytes) - stream->length) return -1;
+ memcpy(stream->bytes + stream->length, data, size);
+ stream->length += size;
+ return 0;
+}
+
+static int memory_read(void *user_data, void *data, size_t size) {
+ MemoryStream *stream = (MemoryStream *)user_data;
+ if (stream->offset > stream->length) return -1;
+ if (size > stream->length - stream->offset) return -1;
+ memcpy(data, stream->bytes + stream->offset, size);
+ stream->offset += size;
+ return 0;
+}
+
+static int fake_snapshot(void *session_impl, ColiSegmentWriteFn write_fn,
+ void *write_user_data,
+ char *error, size_t error_size) {
+ (void)error; (void)error_size;
+ FakeSession *session = (FakeSession *)session_impl;
+ return write_fn(write_user_data, &session->position, sizeof(session->position));
+}
+
+static int fake_restore(void *session_impl, ColiSegmentReadFn read_fn,
+ void *read_user_data,
+ char *error, size_t error_size) {
+ (void)error; (void)error_size;
+ FakeSession *session = (FakeSession *)session_impl;
+ return read_fn(read_user_data, &session->position, sizeof(session->position));
+}
+
+static const ColiSegmentAdapter fake_adapter = {
+ .struct_size = sizeof(fake_adapter),
+ .abi_version = COLI_SEGMENT_ABI_VERSION,
+ .engine_id = "fake",
+ .engine_open = fake_engine_open,
+ .engine_destroy = fake_engine_destroy,
+ .session_create = fake_session_create,
+ .session_destroy = fake_session_destroy,
+ .session_run = fake_session_run,
+ .session_snapshot = fake_snapshot,
+ .session_restore = fake_restore,
+};
+
+static const ColiSegmentAdapter incomplete_snapshot_adapter = {
+ .struct_size = sizeof(incomplete_snapshot_adapter),
+ .abi_version = COLI_SEGMENT_ABI_VERSION,
+ .engine_id = "bad-snapshot",
+ .engine_open = fake_engine_open,
+ .engine_destroy = fake_engine_destroy,
+ .session_create = fake_session_create,
+ .session_destroy = fake_session_destroy,
+ .session_run = fake_session_run,
+ .session_snapshot = fake_snapshot,
+};
+
+static int never_cancel(void *unused) { (void)unused; return 0; }
+static int always_cancel(void *unused) { (void)unused; return 1; }
+
+int main(void) {
+ char error[160] = "";
+ assert(coli_segment_adapter_count() == 0);
+ assert(coli_segment_adapter_register(NULL) != 0);
+ assert(coli_segment_adapter_register(&incomplete_snapshot_adapter) != 0);
+ assert(coli_segment_adapter_register(&fake_adapter) == 0);
+ assert(coli_segment_adapter_register(&fake_adapter) != 0);
+ assert(coli_segment_adapter_count() == 1);
+ assert(coli_segment_adapter_lookup("fake") == &fake_adapter);
+ assert(coli_segment_adapter_lookup("missing") == NULL);
+
+ ColiSegmentEngineOptions open_options = {
+ .struct_size = sizeof(open_options),
+ .model_dir = "unused-fixture",
+ .layer_begin = 1,
+ .layer_end = 4,
+ .context_tokens = 16,
+ };
+ ColiSegmentEngine *engine = NULL;
+ assert(coli_segment_engine_open("fake", &open_options, &engine,
+ error, sizeof(error)) == 0);
+ assert(engine);
+
+ ColiSegmentCapabilities capabilities = {.struct_size = sizeof(capabilities)};
+ assert(coli_segment_engine_capabilities(engine, &capabilities,
+ error, sizeof(error)) == 0);
+ assert(capabilities.state_width == 4 && capabilities.num_layers == 6);
+ assert(capabilities.flags & COLI_SEGMENT_CAP_RANGE_NATIVE);
+ capabilities.struct_size = 0;
+ assert(coli_segment_engine_capabilities(engine, &capabilities,
+ error, sizeof(error)) != 0);
+
+ ColiSegmentSessionOptions session_options = {
+ .struct_size = sizeof(session_options),
+ .context_tokens = 16,
+ };
+ ColiSegmentSession *session = NULL, *second = NULL;
+ assert(coli_segment_session_create(engine, &session_options, &session,
+ error, sizeof(error)) == 0);
+ assert(coli_segment_session_create(engine, &session_options, &second,
+ error, sizeof(error)) == 0);
+ assert(coli_segment_engine_close(engine, error, sizeof(error)) != 0);
+
+ float input[8] = {0, 1, 2, 3, 4, 5, 6, 7};
+ float output[8] = {0};
+ int32_t tokens[2] = {7, 9};
+ ColiSegmentRunRequest run = {
+ .struct_size = sizeof(run),
+ .rows = 2,
+ .position = 0,
+ .token_ids = tokens,
+ .token_count = 2,
+ .input = input,
+ .input_bytes = sizeof(input),
+ .output = output,
+ .output_bytes = sizeof(output),
+ .should_cancel = never_cancel,
+ };
+ assert(coli_segment_run(session, &run, error, sizeof(error)) == 0);
+ for (size_t i = 0; i < 8; i++) assert(output[i] == input[i] + 3.0f);
+
+ MemoryStream stream = {0};
+ assert(coli_segment_snapshot(session, memory_write, &stream,
+ error, sizeof(error)) == 0);
+ assert(stream.length == sizeof(uint64_t));
+ stream.offset = 0;
+ assert(coli_segment_restore(second, memory_read, &stream,
+ error, sizeof(error)) == 0);
+
+ memset(output, 0, sizeof(output));
+ run.position = 2;
+ assert(coli_segment_run(second, &run, error, sizeof(error)) == 0);
+ for (size_t i = 0; i < 8; i++) assert(output[i] == input[i] + 3.0f);
+
+ run.position = 4;
+ run.should_cancel = always_cancel;
+ assert(coli_segment_run(second, &run, error, sizeof(error)) != 0);
+ run.should_cancel = never_cancel;
+ run.token_count = 0;
+ run.token_ids = NULL;
+ assert(coli_segment_run(second, &run, error, sizeof(error)) != 0);
+ run.token_count = 2;
+ assert(coli_segment_run(second, &run, error, sizeof(error)) != 0);
+ run.token_ids = tokens;
+ run.position = 15;
+ assert(coli_segment_run(second, &run, error, sizeof(error)) != 0);
+ run.position = 4;
+ run.input_bytes--;
+ assert(coli_segment_run(second, &run, error, sizeof(error)) != 0);
+
+ coli_segment_session_destroy(second);
+ coli_segment_session_destroy(session);
+ assert(coli_segment_engine_close(engine, error, sizeof(error)) == 0);
+ puts("segment runtime tests: ok");
+ return 0;
+}
diff --git a/docs/segment-runtime.md b/docs/segment-runtime.md
new file mode 100644
index 00000000..87760fe9
--- /dev/null
+++ b/docs/segment-runtime.md
@@ -0,0 +1,63 @@
+# Layer-segment runtime ABI
+
+`c/segment_runtime.h` is Colibri's engine-neutral boundary for callers that
+execute a contiguous, half-open layer range (`begin <= layer < end`). It is a
+local C ABI, not a network protocol. A distributed caller remains responsible
+for peer identity, transport, leases, request IDs, placement and retry policy.
+
+The ABI is deliberately separate from every model's internal structs:
+
+- Colibri owns weights, kernels, accelerator selection and sequence state.
+- An adapter opens only the requested range and exposes the resulting state
+ schema and numeric compatibility class.
+- A caller creates an isolated session for each conversation, sends boundary
+ activations through `coli_segment_run`, and can stream snapshots for
+ migration or recovery.
+
+No engine adapter is registered by this initial API change, so existing CLI,
+server and model behavior is unchanged. Adapters are added and validated per
+model family in later changes.
+
+## Lifecycle
+
+1. Register a static `ColiSegmentAdapter` during process initialization.
+2. Open an engine with `coli_segment_engine_open` and inspect its model-specific
+ `ColiSegmentCapabilities`.
+3. Create one or more sessions. A session must not receive concurrent calls.
+4. Run, snapshot or restore each session as needed.
+5. Destroy every session, then close the engine.
+
+Engine close fails while a session is alive. Registration must finish before
+concurrent lookups begin; registration itself is not a hot-path operation.
+
+## Capability identity
+
+Capabilities are returned after model open because the layer count, boundary
+width and context limit may vary between checkpoints handled by one engine.
+`state_schema` identifies the activation and snapshot layout.
+`numeric_class` identifies builds whose results and snapshots are compatible;
+an adapter must include every relevant precision, reduction and backend rule in
+that class.
+
+`COLI_SEGMENT_CAP_RANGE_NATIVE` is a strong promise: the adapter did not load
+weights outside the requested range. Callers must not publish range-native
+residency when this bit is absent.
+
+## Run contract
+
+Input and output contain exactly `rows * state_width` values in the advertised
+dtype. Token IDs are either absent or contain one entry per row; an adapter can
+make them mandatory with `COLI_SEGMENT_CAP_TOKEN_IDS`. The runtime validates
+sizes and context bounds before calling the adapter.
+
+Positions and model-specific ordering rules remain adapter-owned. A failed or
+cancelled run must not be reported as committed by the distributed caller.
+Network-level idempotency and duplicate request handling belong above this ABI.
+
+## Snapshot contract
+
+Snapshot callbacks stream bytes so neither side needs a second full-state
+allocation. The format is private to an adapter and compatible only when the
+model identity, `state_schema`, numeric class and segment range match. A network
+service should put those fields in its own snapshot envelope before accepting a
+restore.