Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(qwen38): complete bounded prompt runtime

themorlockc50e250324cf
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33128a3d..6c8e042e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,15 @@ Format follows [Keep a Changelog](https://keepachangelog.com/).
- Added family/planner/doctor/build/release integration and a pinned upstream
tiny oracle covering sparse selection, cached decode, LRU eviction, and
sanitizer runs.
+- Kept native FP8 expert payloads and normalized scale banks bounded, added
+ cache-sized parallel demand loading, and made prompt MoE execution
+ expert-major with bounded shared-expert and causal DeltaNet batching.
+- Added exact single-slot hybrid prompt-prefix reuse across QSA, DeltaNet and
+ PLE state, with persistent state and bounded workspace reflected explicitly
+ in planner RAM accounting.
+- Added the boundary-only Qwen3.8 Edge adapter and the seventh-family real
+ Edge -> Segment -> Edge oracle gate without loading transformer, vision or
+ MTP tensors into the Edge process.
## [1.7.0] β€” 2026-08-19
71 pull requests since v1.6.2. A sixth model family with its GPU tier, a
diff --git a/c/Makefile b/c/Makefile
index b6459270..09efea07 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1016,8 +1016,9 @@ qwen38-tiny-generate:
$(PYTHON) tools/make_qwen38_tiny.py --out ./qwen38_tiny
qwen38-tiny-check: qwen38-tiny-generate qwen38$(EXE)
- @for bf16 in 0 1; do for cap in 1 4; do \
- Q38_NATIVE_BF16=$$bf16 OMP_NUM_THREADS=2 SNAP=./qwen38_tiny ./qwen38$(EXE) $$cap 8 ./qwen38_tiny/ref.json || exit $$?; \
+ @for batch in 0 1; do for bf16 in 0 1; do for cap in 1 4; do \
+ Q38_PREFILL_BATCH=$$batch Q38_NATIVE_BF16=$$bf16 OMP_NUM_THREADS=2 SNAP=./qwen38_tiny ./qwen38$(EXE) $$cap 8 ./qwen38_tiny/ref.json || exit $$?; \
+ done; \
done; \
done
@@ -1114,6 +1115,9 @@ tests/test_qwen38_config$(EXE): tests/test_qwen38_config.c qwen38.c qwen38_core.
tests/test_qwen38_serve_framing$(EXE): tests/test_qwen38_serve_framing.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h serve_codec.h tok_unicode.h tok_unicode_o200k.h
$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
+tests/test_qwen38_prefix$(EXE): tests/test_qwen38_prefix.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h serve_codec.h tok_unicode.h tok_unicode_o200k.h
+ $(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
+
tests/test_qwen38_metrics$(EXE): tests/test_qwen38_metrics.c qwen38.c qwen38_core.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h quant.h route_trace.h serve_codec.h tok_unicode.h tok_unicode_o200k.h
$(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)
@@ -1376,7 +1380,8 @@ $(SEGMENT_BUILD_DIR)/qwen36.o: qwen36.c segment_runtime.h edge_runtime.h \
$(CC) $(SEGMENT_CPU_CFLAGS) -DQWEN36_NO_MAIN -c qwen36.c -o $@
$(SEGMENT_BUILD_DIR)/qwen38.o: qwen38.c qwen38_core.h segment_runtime.h \
- segment_adapters.h segment_adapter_internal.h st.h json.h compat.h quant.h \
+ segment_adapters.h segment_adapter_internal.h edge_runtime.h \
+ edge_adapters.h edge_adapter_internal.h st.h json.h compat.h quant.h \
qwen38_nfc.h qwen38_nfc_tables.h route_trace.h tok_unicode.h \
tok_unicode_o200k.h | $(SEGMENT_BUILD_DIR)
$(CC) $(SEGMENT_CPU_CFLAGS) -DQWEN38_NO_MAIN -c qwen38.c -o $@
@@ -1469,12 +1474,14 @@ edge-adapters-real: tests/test_edge_adapters_real$(EXE)
@test -n "$(KIMI_EDGE_MODEL)" && test -n "$(KIMI_EDGE_REF)" || { echo "set KIMI_EDGE_MODEL and KIMI_EDGE_REF" >&2; exit 2; }
@test -n "$(OLMOE_EDGE_MODEL)" && test -n "$(OLMOE_EDGE_REF)" || { echo "set OLMOE_EDGE_MODEL and OLMOE_EDGE_REF" >&2; exit 2; }
@test -n "$(QWEN_EDGE_MODEL)" && test -n "$(QWEN_EDGE_REF)" || { echo "set QWEN_EDGE_MODEL and QWEN_EDGE_REF" >&2; exit 2; }
+ @test -n "$(QWEN38_EDGE_MODEL)" && test -n "$(QWEN38_EDGE_REF)" || { echo "set QWEN38_EDGE_MODEL and QWEN38_EDGE_REF" >&2; exit 2; }
@test -n "$(DEEPSEEK_EDGE_MODEL)" && test -n "$(DEEPSEEK_EDGE_REF)" || { echo "set DEEPSEEK_EDGE_MODEL and DEEPSEEK_EDGE_REF" >&2; exit 2; }
GLM_SEGMENT_EBITS=16 GLM_SEGMENT_DBITS=16 OMP_NUM_THREADS=$${OMP_NUM_THREADS:-2} ./tests/test_edge_adapters_real$(EXE) glm "$(GLM_EDGE_MODEL)" "$(GLM_EDGE_REF)" 3
INK_SEGMENT_BITS=0 OMP_NUM_THREADS=$${OMP_NUM_THREADS:-2} ./tests/test_edge_adapters_real$(EXE) inkling "$(INKLING_EDGE_MODEL)" "$(INKLING_EDGE_REF)" 3
COLI_RAM_OVERCOMMIT=1 K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 OMP_NUM_THREADS=$${OMP_NUM_THREADS:-2} ./tests/test_edge_adapters_real$(EXE) kimi "$(KIMI_EDGE_MODEL)" "$(KIMI_EDGE_REF)" 3
OMP_NUM_THREADS=$${OMP_NUM_THREADS:-2} ./tests/test_edge_adapters_real$(EXE) olmoe "$(OLMOE_EDGE_MODEL)" "$(OLMOE_EDGE_REF)" 3
OMP_NUM_THREADS=$${OMP_NUM_THREADS:-2} ./tests/test_edge_adapters_real$(EXE) qwen36 "$(QWEN_EDGE_MODEL)" "$(QWEN_EDGE_REF)" 3
+ OMP_NUM_THREADS=$${OMP_NUM_THREADS:-2} ./tests/test_edge_adapters_real$(EXE) qwen38 "$(QWEN38_EDGE_MODEL)" "$(QWEN38_EDGE_REF)" 3
OMP_NUM_THREADS=$${OMP_NUM_THREADS:-2} ./tests/test_edge_adapters_real$(EXE) deepseek_v4 "$(DEEPSEEK_EDGE_MODEL)" "$(DEEPSEEK_EDGE_REF)" 3
V4_HOT_TEST_OBJ = COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16_TEST.o
diff --git a/c/edge_adapters.h b/c/edge_adapters.h
index 059aa746..3961e6c0 100644
--- a/c/edge_adapters.h
+++ b/c/edge_adapters.h
@@ -13,6 +13,7 @@ int coli_inkling_edge_adapter_register(void);
int coli_kimi_edge_adapter_register(void);
int coli_olmoe_edge_adapter_register(void);
int coli_qwen36_edge_adapter_register(void);
+int coli_qwen38_edge_adapter_register(void);
int coli_deepseek_v4_edge_adapter_register(void);
#ifdef __cplusplus
diff --git a/c/family_registry.py b/c/family_registry.py
index dd0b004c..0733eeb0 100644
--- a/c/family_registry.py
+++ b/c/family_registry.py
@@ -78,6 +78,12 @@ class FamilyDescriptor:
has_cli_adapter: bool = False
tune_prompt_template: str = "{prompt}"
supports_accelerator: bool = True
+ # Optional model-owned allocations that are neither dense resident
+ # tensors nor per-expert cache entries. Qwen3.8 uses this for the
+ # normalized FP8 scale bank shared by every cache slot. Keeping this
+ # separate from expert_inventory prevents a fixed allocation from
+ # being multiplied by the cache capacity.
+ fixed_resident_inventory: object = None
@dataclass(frozen=True, slots=True)
@@ -152,6 +158,19 @@ def _qwen36_geometry(config, context, _model_dir):
return PlannerGeometry(kv, fixed, 0, _required_int(config, "num_experts", "qwen36"))
+_QWEN38_PREFILL_BATCH_ROWS = 32
+_QWEN38_PREFILL_WORKSPACE_BYTES = 64 << 20
+
+
+def _qwen38_bounded_prefill(rows, fixed_bytes, row_bytes):
+ """Mirror q38_bounded_prefill_rows() without context-sized scratch."""
+ selected = min(rows, _QWEN38_PREFILL_BATCH_ROWS)
+ while selected > 1 and fixed_bytes + selected * row_bytes > \
+ _QWEN38_PREFILL_WORKSPACE_BYTES:
+ selected -= 1
+ return selected
+
+
def _qwen38_geometry(config, context, _model_dir):
"""Qwen3.8-Flash-Next's hybrid state layout.
@@ -242,6 +261,16 @@ def _qwen38_geometry(config, context, _model_dir):
fixed += len(ple_ids) * (hc_width * ngram * ple_conv * 4 +
(ngram - 1) * 8)
+ # The one-slot serve path keeps an exact prompt-prefix checkpoint. QSA rows
+ # remain in their existing append-only cache, while every recurrent/PLE
+ # payload is copied once so decode can mutate the live state without
+ # damaging the reusable prompt. It also retains the prompt token IDs and
+ # final prompt logits. The adapter/runtime base reserve covers allocator and
+ # pointer metadata; these are the complete model-sized payloads.
+ prefix_snapshot = fixed + vocab * 4
+ fixed += prefix_snapshot
+ context_state += context * 4
+
# Match the peak buffers held concurrently by step(): the persistent
# hyper/mixed/inject/block rows plus the largest layer-local row set. The
# base planner reserve still covers allocator and non-scaling overhead.
@@ -266,6 +295,30 @@ def _qwen38_geometry(config, context, _model_dir):
head_dim + index_budget + index_ratio - 1))
workspace = (context * max(gated_residual_row, qsa_row, ple_row) +
max(moe_fixed, gdn_fixed, ple_fixed, qsa_fixed)) * 4
+
+ # q38_moe_prefill() and q38_deltanet() batch only a bounded row chunk.
+ # Account their exact simultaneous allocations beside the context-sized
+ # outer hyper/mixed/inject/block rows. Q38RouteAssignment is one int plus
+ # one float (8 bytes), and supported hosts use 64-bit pointers.
+ moe_assignment_bytes = 2 * (hidden + intermediate) * 4 + 8 + 2 * 4
+ moe_row_bytes = (experts + 3 * shared + hidden + 1) * 4 + \
+ topk * moe_assignment_bytes
+ moe_fixed_bytes = experts * (4 * 4 + 8) + 4
+ moe_rows = _qwen38_bounded_prefill(
+ context, moe_fixed_bytes, moe_row_bytes)
+ moe_chunk_bytes = moe_fixed_bytes + moe_rows * moe_row_bytes
+
+ delta_row_bytes = (gdn_conv_width + 2 * value_heads * value_dim +
+ 2 * value_heads) * 4
+ delta_fixed_bytes = (gdn_conv_width +
+ 2 * value_heads * key_dim +
+ value_heads * value_dim) * 4
+ delta_rows = _qwen38_bounded_prefill(
+ context, delta_fixed_bytes, delta_row_bytes)
+ delta_chunk_bytes = delta_fixed_bytes + delta_rows * delta_row_bytes
+ prefill_workspace = context * base_row * 4 + max(
+ moe_chunk_bytes, delta_chunk_bytes)
+ workspace = max(workspace, prefill_workspace)
return PlannerGeometry(context_state, fixed, workspace,
experts)
@@ -563,11 +616,13 @@ def _inkling_expert_inventory(name, size, config, _dtype=None):
def _qwen38_expert_inventory(name, size, config, dtype=None):
- """Index both per-expert FP8 and fused BF16 expert tensor layouts.
+ """Index per-expert FP8 and fused BF16 expert tensor layouts.
Resource planning calls this once per tensor; summing the returned byte
counts therefore combines gate/up and down tensors for each expert without
- multiplying either tensor by the expert count a second time.
+ multiplying either tensor by the expert count a second time. FP8 scale
+ sidecars are validated here but accounted for by the fixed resident
+ inventory because the native loader shares their normalized bank.
"""
match = _QWEN38_EXPERT.fullmatch(name)
if match:
@@ -592,26 +647,11 @@ def _qwen38_expert_inventory(name, size, config, dtype=None):
return ((int(match.group(1)), int(match.group(2)), retained),)
match = _QWEN38_EXPERT_SCALE.fullmatch(name)
if match:
- # Slots normalize every supported source scale dtype to F32. Derive
- # the retained grid rather than charging the source byte count (the
- # official sidecars happen to be BF16).
- hidden = _required_int(config, "hidden_size", "qwen38")
- intermediate = _required_int(config, "moe_intermediate_size", "qwen38")
- projection = match.group(3)
- rows = intermediate if projection != "down_proj" else hidden
- cols = hidden if projection != "down_proj" else intermediate
- scale_count = math.ceil(rows / 128) * math.ceil(cols / 128)
- if dtype is None:
- if size == scale_count * 4:
- dtype = "F32"
- elif size == scale_count * 2:
- dtype = "BF16"
- source_bytes = {"BF16": 2, "F16": 2, "F32": 4}.get(dtype)
- if source_bytes is None or size != scale_count * source_bytes:
- raise ValueError(f"qwen38: expert scale {name!r} has unsupported "
- f"dtype/size {dtype}/{size}")
- scale_bytes = scale_count * 4
- return ((int(match.group(1)), int(match.group(2)), scale_bytes),)
+ # The native loader normalizes all scales into one fixed per-layer
+ # bank. They are deliberately excluded from per-slot accounting;
+ # _qwen38_fixed_resident_inventory() accounts for that bank once.
+ _qwen38_scale_bytes(name, size, config, dtype)
+ return ()
match = _QWEN38_FUSED_EXPERT.fullmatch(name)
if not match:
return ()
@@ -634,6 +674,47 @@ def _qwen38_expert_inventory(name, size, config, dtype=None):
return tuple((layer, expert, per_expert) for expert in range(experts))
+def _qwen38_scale_bytes(name, size, config, dtype=None):
+ """Validate a Qwen3.8 FP8 scale sidecar and return its retained bytes.
+
+ The source sidecar may be BF16 or F32, while the native engine always
+ retains the decoded values as F32. This helper is shared by the expert
+ inventory and fixed-resident inventory so validation and sizing cannot
+ drift apart.
+ """
+ match = _QWEN38_EXPERT_SCALE.fullmatch(name)
+ if match is None:
+ return 0
+ hidden = _required_int(config, "hidden_size", "qwen38")
+ intermediate = _required_int(config, "moe_intermediate_size", "qwen38")
+ projection = match.group(3)
+ rows = intermediate if projection != "down_proj" else hidden
+ cols = hidden if projection != "down_proj" else intermediate
+ scale_count = math.ceil(rows / 128) * math.ceil(cols / 128)
+ if dtype is None:
+ if size == scale_count * 4:
+ dtype = "F32"
+ elif size == scale_count * 2:
+ dtype = "BF16"
+ source_bytes = {"BF16": 2, "F16": 2, "F32": 4}.get(dtype)
+ if source_bytes is None or size != scale_count * source_bytes:
+ raise ValueError(f"qwen38: expert scale {name!r} has unsupported "
+ f"dtype/size {dtype}/{size}")
+ return scale_count * 4
+
+
+def _qwen38_fixed_resident_inventory(name, size, config, dtype=None):
+ """Return bytes for the native loader's fixed normalized scale bank.
+
+ A sidecar contributes its normalized F32 footprint exactly once. For a
+ native official checkpoint, summing all sidecars produces the complete
+ bank (layers x experts x three projections), independent of cache slots.
+ For a fallback layout the same charge is conservative: expanded loaders
+ do not retain the bank, but reserving it cannot understate memory.
+ """
+ return _qwen38_scale_bytes(name, size, config, dtype)
+
+
_QWEN38_NATIVE_MATRIX_SUFFIXES = (
"embed_tokens.weight",
"input_mix_weight_down.weight",
@@ -665,6 +746,10 @@ def _qwen38_resident_inventory(name, size, _config, dtype=None):
rank-two matrices stay in that two-byte representation; the comparatively
small norms, gates and convolution vectors still expand through st_read_f32.
"""
+ if _QWEN38_EXPERT_SCALE.fullmatch(name):
+ # Expert scales belong to the fixed normalized bank, not dense
+ # residency. The dedicated inventory hook accounts them once.
+ return 0
if name.startswith("mtp.") or name.startswith("model.visual."):
return 0
if ".ple.ple_embedding.ngram_embedding." in name and name.endswith(".weight"):
@@ -852,6 +937,7 @@ FAMILIES = (
planner_unsupported_reason="",
expert_inventory=_qwen38_expert_inventory,
resident_inventory=_qwen38_resident_inventory,
+ fixed_resident_inventory=_qwen38_fixed_resident_inventory,
config_section="text_config",
limits=FamilyLimits(8192, 262144, 1024, 8192, 1, 1, "Q38_MAXT"),
capabilities=FamilyCapabilities(False, False, False, True),
@@ -907,6 +993,8 @@ def _build_registry(families):
not callable(family.expert_inventory) or
(family.resident_inventory is not None and
not callable(family.resident_inventory)) or
+ (family.fixed_resident_inventory is not None and
+ not callable(family.fixed_resident_inventory)) or
not isinstance(family.supports_accelerator, bool) or
not isinstance(family.has_gateway_adapter, bool) or
not isinstance(family.has_cli_adapter, bool) or
@@ -1041,6 +1129,25 @@ def resident_contribution(resolved, name, size, dtype=None):
return contribution
+def fixed_resident_contribution(resolved, name, size, dtype=None):
+ """Return resident bytes that do not belong to dense or cache storage.
+
+ This pool is model-owned and independent of cache capacity. Families
+ without such an allocation return zero.
+ """
+ if isinstance(size, bool) or not isinstance(size, int) or size < 0:
+ raise ValueError("tensor size must be a non-negative integer")
+ if dtype is not None and not isinstance(dtype, str):
+ raise ValueError("tensor dtype must be a string")
+ inventory = resolved.descriptor.fixed_resident_inventory
+ contribution = 0 if inventory is None else inventory(
+ name, size, resolved.family_config, dtype)
+ if isinstance(contribution, bool) or not isinstance(contribution, int) or contribution < 0:
+ raise RegistryError(
+ f"invalid fixed resident inventory for {resolved.descriptor.id}")
+ return contribution
+
+
def public_metadata(family):
return {
"id": family.id,
diff --git a/c/qwen38.c b/c/qwen38.c
index 244c4ef4..6571b43c 100644
--- a/c/qwen38.c
+++ b/c/qwen38.c
@@ -451,7 +451,7 @@ static int load_tokenizer(const char *path){
}
jval *normalizer=json_get(root,"normalizer");
- if(normalizer){
+ if(normalizer&&normalizer->t!=J_NULL){
const char *kind=normalizer->t==J_OBJ?jstr(normalizer,"type"):NULL;
if(!kind||strcmp(kind,"NFC")){
fprintf(stderr,"[tok] unsupported tokenizer normalizer in %s\n",path);goto cleanup;
@@ -915,6 +915,11 @@ static void emit_openai_result(const int *out, int np, int n_new, int stream){
#include "segment_adapters.h"
#include "segment_adapter_internal.h"
#endif
+#ifdef COLI_EDGE_ADAPTER
+#include "edge_runtime.h"
+#include "edge_adapters.h"
+#include "edge_adapter_internal.h"
+#endif
/* Qwen3.8 routes every transformer layer and has no MTP row. Keep the
* process-global route_trace owner in the CLI/serve model only; segment
@@ -1102,6 +1107,8 @@ static int compare_reference_logits(const float *actual,const jval *expected){
* engine: \x01\x01READY\x01\x01\n
* STAT 0 0.00 0.0 <rss>\n
* gateway: SUBMIT <id> <slot> <plen> <max_tok> <temp> <top_p>\n <payload bytes>\n
+ * Qwen3.8 accepts slot 0 only; its single hybrid context is reused when
+ * the next prompt begins with the exact token sequence it already covers.
* engine: ACCEPT <id> <np>\n
* DATA <id> <n>\n <bytes>\n (repeated per decoded chunk)
* DONE <id> STAT <gen> <tps> <hit%> <rss> <np> <limited>\n
@@ -1111,11 +1118,18 @@ static int compare_reference_logits(const float *actual,const jval *expected){
* session hangs forever (#748). compat.h's coli_serve_binary_mode (#749)
* carries that fix for every engine; see its comment for the full story. */
-typedef struct { char id[64]; int max_tok; float temp, top_p; char *payload; int plen; } ServeReq;
+typedef struct {
+ char id[64];
+ int slot;
+ int max_tok;
+ float temp, top_p;
+ char *payload;
+ int plen;
+} ServeReq;
static const ColiServeWireProfile q38_wire = {
.max_header_bytes = 511,
.max_payload_bytes = 1u << 24,
- .max_tokens = INT_MAX,
+ .max_tokens = 1 << 20,
.require_exact_lf = 1,
.require_finite_sampling = 1,
};
@@ -1144,8 +1158,16 @@ static int serve_read_req(FILE *in,FILE *out,ServeReq *q,const char *active_id){
coli_serve_write_error(out,command.id,"engine busy");
coli_serve_command_dispose(&command);return 0;
}
+ /* Qwen3.8 owns one recurrent/KV state. The gateway still sends the
+ * common cache-slot field, so reject every slot except that sole slot
+ * after the codec has consumed the complete frame. */
+ if(command.slot!=0){
+ coli_serve_write_error(out,command.id,"invalid cache slot");
+ coli_serve_command_dispose(&command);return 0;
+ }
if(!q){coli_serve_command_dispose(&command);return 0;}
snprintf(q->id,sizeof(q->id),"%s",command.id);
+ q->slot=command.slot;
q->max_tok=command.max_tokens;q->temp=command.temperature;q->top_p=command.top_p;
q->payload=(char*)coli_serve_command_take_payload(&command);q->plen=(int)command.payload_bytes;
coli_serve_command_dispose(&command);return 2;
@@ -1200,6 +1222,236 @@ static int serve_eos_ids(int *ids,int cap,int config_eos,int vocab){
return n;
}
+/* ---------- one-slot hybrid prompt prefix -----------------------------
+ *
+ * Qwen3.8 has two different kinds of state. DeltaNet and PLE carry a
+ * recurrent state which generation mutates in place; QSA carries position-
+ * indexed K/V/index rows which are append-only. A prompt cache therefore
+ * keeps a bounded snapshot of only the recurrent state and leaves the QSA
+ * rows in the model's existing buffers. Generation writes rows strictly
+ * after the cached prompt, so those rows remain valid for the next turn.
+ *
+ * There is deliberately one process-local cache: the Qwen serve ABI exposes
+ * one slot and the serve loop owns one Model. This is not a second KV cache
+ * and cannot grow with the number of requests. The snapshot is published
+ * only after the complete prompt has been evaluated; a cancelled/stopped
+ * decode consequently cannot damage the prompt state that follows it. */
+typedef struct {
+ Model *owner;
+ int valid;
+ int layers;
+ int vocab;
+ size_t rec_cells;
+ size_t conv_cells;
+ size_t ple_cells;
+ int *ids;
+ int id_cap;
+ int len;
+ float *logits;
+ float **dn_rec;
+ float **dn_conv;
+ float *ple_conv;
+ int64_t ple_history[2];
+ int ple_history_len;
+} Q38PrefixCache;
+
+static Q38PrefixCache g_q38_prefix;
+
+static int q38_size_mul(size_t left,size_t right,size_t *out){
+ if(!out||(right&&left>SIZE_MAX/right))return 0;
+ *out=left*right;return 1;
+}
+
+static int q38_prefix_geometry(const Model *m,size_t *rec_cells,
+ size_t *conv_cells,size_t *ple_cells){
+ if(!m||!rec_cells||!conv_cells||!ple_cells||m->c.layers<1||
+ m->c.layers>Q38_MAX_LAYERS||m->c.vocab<1||!m->c.is_attn)return 0;
+ const Cfg *c=&m->c;size_t rec=0,conv=0,ple=0,cells=0;
+ if(c->dn_vheads<=0||c->dn_kdim<=0||c->dn_vdim<=0||
+ c->dn_conv_dim<=0||c->dn_convk<2)return 0;
+ if(!q38_size_mul((size_t)c->dn_vheads,(size_t)c->dn_kdim,&cells)||
+ !q38_size_mul(cells,(size_t)c->dn_vdim,&cells))return 0;
+ rec=cells;
+ if(!q38_size_mul((size_t)c->dn_conv_dim,(size_t)(c->dn_convk-1),&cells))return 0;
+ conv=cells;
+ if(c->ple_layer>=0&&c->ple_layer<c->layers){
+ if(c->hc_width<=0||c->ple_convk<2||c->ngram_size<=0||
+ !q38_size_mul((size_t)c->hc_width,(size_t)(c->ple_convk-1),&cells)||
+ !q38_size_mul(cells,(size_t)c->ngram_size,&ple))return 0;
+ if(!m->PLE_conv_state||!m->ple_history)return 0;
+ }
+ *rec_cells=rec;*conv_cells=conv;*ple_cells=ple;return 1;
+}
+
+static void q38_prefix_cache_dispose(Q38PrefixCache *cache){
+ if(!cache)return;
+ if(cache->dn_rec)for(int i=0;i<cache->layers;i++)free(cache->dn_rec[i]);
+ if(cache->dn_conv)for(int i=0;i<cache->layers;i++)free(cache->dn_conv[i]);
+ free(cache->dn_rec);free(cache->dn_conv);free(cache->ple_conv);
+ free(cache->ids);free(cache->logits);memset(cache,0,sizeof(*cache));
+}
+
+static void q38_prefix_cache_invalidate(void){
+ g_q38_prefix.valid=0;g_q38_prefix.len=0;
+}
+
+static int q38_prefix_cache_layout(Model *m){
+ if(!m)return 0;
+ size_t rec=0,conv=0,ple=0;
+ if(!q38_prefix_geometry(m,&rec,&conv,&ple))return 0;
+ if(g_q38_prefix.owner!=m||g_q38_prefix.layers!=m->c.layers||
+ g_q38_prefix.vocab!=m->c.vocab||g_q38_prefix.rec_cells!=rec||
+ g_q38_prefix.conv_cells!=conv||g_q38_prefix.ple_cells!=ple){
+ q38_prefix_cache_dispose(&g_q38_prefix);g_q38_prefix.owner=m;
+ g_q38_prefix.layers=m->c.layers;g_q38_prefix.vocab=m->c.vocab;
+ g_q38_prefix.rec_cells=rec;g_q38_prefix.conv_cells=conv;
+ g_q38_prefix.ple_cells=ple;
+ }
+ if(g_q38_prefix.dn_rec&&g_q38_prefix.dn_conv&&g_q38_prefix.logits&&
+ (!ple||g_q38_prefix.ple_conv))return 1;
+ size_t logits_bytes=0;
+ if(!q38_size_mul((size_t)m->c.vocab,sizeof(float),&logits_bytes))return 0;
+ g_q38_prefix.dn_rec=(float**)calloc((size_t)m->c.layers,sizeof(float*));
+ g_q38_prefix.dn_conv=(float**)calloc((size_t)m->c.layers,sizeof(float*));
+ g_q38_prefix.logits=(float*)malloc(logits_bytes);
+ if(!g_q38_prefix.dn_rec||!g_q38_prefix.dn_conv||!g_q38_prefix.logits){
+ q38_prefix_cache_dispose(&g_q38_prefix);return 0;
+ }
+ for(int i=0;i<m->c.layers;i++)if(!m->c.is_attn[i]){
+ if(!m->DN_rec[i]||!m->DN_conv[i]||
+ (g_q38_prefix.rec_cells>SIZE_MAX/sizeof(float))||
+ (g_q38_prefix.conv_cells>SIZE_MAX/sizeof(float)))
+ {q38_prefix_cache_dispose(&g_q38_prefix);return 0;}
+ g_q38_prefix.dn_rec[i]=(float*)malloc(g_q38_prefix.rec_cells*sizeof(float));
+ g_q38_prefix.dn_conv[i]=(float*)malloc(g_q38_prefix.conv_cells*sizeof(float));
+ if(!g_q38_prefix.dn_rec[i]||!g_q38_prefix.dn_conv[i]){
+ q38_prefix_cache_dispose(&g_q38_prefix);return 0;
+ }
+ }
+ if(ple){
+ if(g_q38_prefix.ple_cells>SIZE_MAX/sizeof(float)){
+ q38_prefix_cache_dispose(&g_q38_prefix);return 0;
+ }
+ g_q38_prefix.ple_conv=(float*)malloc(g_q38_prefix.ple_cells*sizeof(float));
+ if(!g_q38_prefix.ple_conv){q38_prefix_cache_dispose(&g_q38_prefix);return 0;}
+ }
+ return 1;
+}
+
+static int q38_prefix_ids_reserve(int len){
+ if(len<1||(size_t)len>SIZE_MAX/sizeof(int))return 0;
+ if(g_q38_prefix.ids&&len<=g_q38_prefix.id_cap)return 1;
+ size_t bytes=(size_t)len*sizeof(int);int *ids=(int*)malloc(bytes);
+ if(!ids)return 0;
+ if(g_q38_prefix.ids&&g_q38_prefix.len>0)
+ memcpy(ids,g_q38_prefix.ids,(size_t)g_q38_prefix.len*sizeof(int));
+ free(g_q38_prefix.ids);g_q38_prefix.ids=ids;g_q38_prefix.id_cap=len;return 1;
+}
+
+static void q38_prefix_copy_state(Model *m,int to_cache){
+ for(int i=0;i<m->c.layers;i++)if(!m->c.is_attn[i]){
+ float *rec=to_cache?g_q38_prefix.dn_rec[i]:m->DN_rec[i];
+ float *conv=to_cache?g_q38_prefix.dn_conv[i]:m->DN_conv[i];
+ const float *src_rec=to_cache?m->DN_rec[i]:g_q38_prefix.dn_rec[i];
+ const float *src_conv=to_cache?m->DN_conv[i]:g_q38_prefix.dn_conv[i];
+ memcpy(rec,src_rec,g_q38_prefix.rec_cells*sizeof(float));
+ memcpy(conv,src_conv,g_q38_prefix.conv_cells*sizeof(float));
+ }
+ if(g_q38_prefix.ple_cells){
+ if(to_cache){
+ memcpy(g_q38_prefix.ple_conv,m->PLE_conv_state,
+ g_q38_prefix.ple_cells*sizeof(float));
+ g_q38_prefix.ple_history_len=m->ple_history_len;
+ memcpy(g_q38_prefix.ple_history,m->ple_history,sizeof(g_q38_prefix.ple_history));
+ }else{
+ memcpy(m->PLE_conv_state,g_q38_prefix.ple_conv,
+ g_q38_prefix.ple_cells*sizeof(float));
+ m->ple_history_len=g_q38_prefix.ple_history_len;
+ memcpy(m->ple_history,g_q38_prefix.ple_history,sizeof(g_q38_prefix.ple_history));
+ }
+ }
+}
+
+static int q38_prefix_cache_save(Model *m,const int *ids,int len,const float *logits){
+ if(!m||!ids||len<1||!logits||!q38_prefix_cache_layout(m)||
+ !q38_prefix_ids_reserve(len))return 0;
+ memcpy(g_q38_prefix.ids,ids,(size_t)len*sizeof(int));
+ memcpy(g_q38_prefix.logits,logits,(size_t)m->c.vocab*sizeof(float));
+ q38_prefix_copy_state(m,1);g_q38_prefix.len=len;g_q38_prefix.valid=1;return 1;
+}
+
+/* Restore only when the complete cached prompt is an exact prefix. A shorter
+ * request would require rewinding append-only QSA rows and is therefore a
+ * safe miss; equality is useful because the saved prompt logits let the
+ * caller enter decode without feeding the final prompt token twice. */
+static int q38_prefix_restore(Model *m,const int *ids,int len){
+ if(!m||!ids||len<1||!g_q38_prefix.valid||g_q38_prefix.owner!=m||
+ g_q38_prefix.len<1||g_q38_prefix.len>len||
+ memcmp(g_q38_prefix.ids,ids,(size_t)g_q38_prefix.len*sizeof(int)))return 0;
+ q38_prefix_copy_state(m,0);m->kv_len=g_q38_prefix.len;return g_q38_prefix.len;
+}
+
+static const float *q38_prefix_cached_logits(Model *m){
+ return m&&g_q38_prefix.valid&&g_q38_prefix.owner==m?g_q38_prefix.logits:NULL;
+}
+
+static void q38_prefix_cache_release(Model *m){
+ if(!m||g_q38_prefix.owner!=m)return;
+ q38_prefix_cache_dispose(&g_q38_prefix);
+}
+
+/* Grow QSA storage without disturbing rows already described by the prefix.
+ * The core's ordinary allocator intentionally frees on growth for cold
+ * inference; serve needs this transactional variant for a cached extension. */
+static int q38_serve_ensure_kv(Model *m,int required){
+ if(!m||required<1)return 0;
+ if(m->K&&required<=m->kv_cap){m->max_t=required;return 1;}
+ int old_cap=m->kv_cap,old_max=m->max_t,keep=m->kv_len;
+ if(!m->K||old_cap<1||keep<1){m->max_t=required;ensure_kv(m);return 1;}
+ if(keep>old_cap)keep=old_cap;if(keep>required)keep=required;
+ Cfg *c=&m->c;float **new_k=(float**)calloc((size_t)c->layers,sizeof(float*));
+ float **new_v=(float**)calloc((size_t)c->layers,sizeof(float*));
+ float **new_ik=(float**)calloc((size_t)c->layers,sizeof(float*));
+ if(!new_k||!new_v||!new_ik)goto oom;
+ size_t cells=0;
+ for(int i=0;i<c->layers;i++)if(c->is_attn[i]){
+ if(c->kv_heads<=0||c->head_dim<=0||c->idx_dim<=0||
+ !m->K[i]||!m->V[i]||!m->IK[i]||
+ !q38_size_mul((size_t)c->kv_heads,(size_t)required,&cells)||
+ !q38_size_mul(cells,(size_t)c->head_dim,&cells)||
+ cells>SIZE_MAX/sizeof(float))goto oom;
+ new_k[i]=(float*)malloc(cells*sizeof(float));new_v[i]=(float*)malloc(cells*sizeof(float));
+ if(!new_k[i]||!new_v[i])goto oom;
+ size_t row_cells=0,row_bytes=0;
+ if(!q38_size_mul((size_t)keep,(size_t)c->head_dim,&row_cells)||
+ !q38_size_mul(row_cells,sizeof(float),&row_bytes))goto oom;
+ for(int h=0;h<c->kv_heads;h++){
+ size_t old_off=0,new_off=0;
+ if(!q38_size_mul((size_t)h,(size_t)old_cap,&old_off)||
+ !q38_size_mul(old_off,(size_t)c->head_dim,&old_off)||
+ !q38_size_mul((size_t)h,(size_t)required,&new_off)||
+ !q38_size_mul(new_off,(size_t)c->head_dim,&new_off))goto oom;
+ memcpy(new_k[i]+new_off,m->K[i]+old_off,row_bytes);
+ memcpy(new_v[i]+new_off,m->V[i]+old_off,row_bytes);
+ }
+ if(!q38_size_mul((size_t)required,(size_t)c->idx_dim,&cells)||
+ cells>SIZE_MAX/sizeof(float))goto oom;
+ new_ik[i]=(float*)malloc(cells*sizeof(float));if(!new_ik[i])goto oom;
+ size_t index_cells=0,index_bytes=0;
+ if(!q38_size_mul((size_t)keep,(size_t)c->idx_dim,&index_cells)||
+ !q38_size_mul(index_cells,sizeof(float),&index_bytes))goto oom;
+ memcpy(new_ik[i],m->IK[i],index_bytes);
+ }
+ for(int i=0;i<c->layers;i++){
+ free(m->K[i]);free(m->V[i]);free(m->IK[i]);
+ }
+ free(m->K);free(m->V);free(m->IK);m->K=new_k;m->V=new_v;m->IK=new_ik;
+ m->kv_cap=required;m->max_t=required;return 1;
+oom:
+ if(new_k)for(int i=0;i<c->layers;i++){free(new_k[i]);free(new_v?new_v[i]:NULL);free(new_ik?new_ik[i]:NULL);}
+ free(new_k);free(new_v);free(new_ik);m->max_t=old_max;return 0;
+}
+
static double q38_cache_hit_percent(uint64_t hits,uint64_t misses){
double total=(double)hits+(double)misses;
return total?100.0*(double)hits/total:0.0;
@@ -1228,7 +1480,7 @@ static int serve_one(Model *m, ServeReq *q){
encode_text_n(q->payload,(size_t)q->plen,&ids,&np); /* byte-counted prompt; qwen38 adds no BOS */
int max_ctx = qwen38_max_ctx();
if(max_ctx > m->c.max_positions) max_ctx = m->c.max_positions;
- if(np<1 || np>max_ctx || q->max_tok>max_ctx-np){
+ if(np<1 || np>max_ctx || q->max_tok<1 || q->max_tok>max_ctx-np){
printf("ERROR %s CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d\n",q->id,np,q->max_tok,max_ctx);
fflush(stdout); free(ids); return 0;
}
@@ -1236,9 +1488,34 @@ static int serve_one(Model *m, ServeReq *q){
double request_started=now_s();
uint64_t hits_before=m->hits, misses_before=m->miss;
Q38Timers timers_before=m->timers;
- m->max_t = np + q->max_tok;
- reset_recurrent(m); ensure_kv(m); m->kv_len = 0;
- float *lo = step(m, ids, np, 0);
+ int request_capacity=np+q->max_tok;
+ if(!q38_serve_ensure_kv(m,request_capacity)){
+ coli_serve_write_error(stdout,q->id,"unable to grow QSA state");
+ free(ids);return 0;
+ }
+ int reuse=q38_prefix_restore(m,ids,np);
+ float *lo=NULL;
+ if(reuse==np){
+ const float *cached=q38_prefix_cached_logits(m);
+ if(!cached){
+ q38_prefix_cache_invalidate();reset_recurrent(m);m->kv_len=0;
+ lo=step(m,ids,np,0);reuse=0;
+ }else{
+ lo=falloc(m->c.vocab);
+ memcpy(lo,cached,(size_t)m->c.vocab*sizeof(float));
+ }
+ }else if(reuse>0){
+ lo=step(m,ids+reuse,np-reuse,reuse);
+ }else{
+ /* A miss invalidates the snapshot before reset so no later request can
+ * pair its recurrent state with the old token record. */
+ q38_prefix_cache_invalidate();reset_recurrent(m);m->kv_len=0;
+ lo=step(m,ids,np,0);
+ }
+ if(reuse!=np&&!q38_prefix_cache_save(m,ids,np,lo)&&getenv("Q38_PREFIX_LOG"))
+ fprintf(stderr,"[qwen38 prefix] cache disabled for request %s (state snapshot unavailable)\n",q->id);
+ if(getenv("Q38_PREFIX_LOG"))
+ fprintf(stderr,"[qwen38 prefix] request=%s reused=%d/%d\n",q->id,reuse,np);
int gen=0, limited=1, cancelled=0, stopped=0, input_eof=0;
int eos_ids[4];int n_eos=serve_eos_ids(eos_ids,4,m->c.eos_id,m->c.vocab);
double first_token_at=0.0,last_token_at=0.0;
@@ -1277,8 +1554,9 @@ static int serve_one(Model *m, ServeReq *q){
}
if(cancelled||stopped){limited=0;break;}
/* The next logits are not needed after the final requested token.
- * serve_one() resets the recurrent/KV state for every request, so
- * stepping here would only run a full discarded decode pass. */
+ * The published prompt snapshot, rather than the live decode state,
+ * is restored on the next request, so stepping here would only run a
+ * full discarded decode pass. */
if(s == q->max_tok - 1) break;
lo = step(m, &tk, 1, np+s);
}
@@ -1317,8 +1595,11 @@ static void serve_loop(Model *m){
for(;;){
ServeReq q={0}; int r;
do r=serve_read_req(stdin,stdout,&q,NULL); while(r!=2&&r>=0);
- if(r<0) return;
- if(r==2){ int status=serve_one(m,&q); free(q.payload); if(status<0)return; }
+ if(r<0){q38_prefix_cache_release(m);return;}
+ if(r==2){
+ int status=serve_one(m,&q);free(q.payload);
+ if(status<0){q38_prefix_cache_release(m);return;}
+ }
}
}
@@ -1450,6 +1731,7 @@ int main(int argc, char **argv) {
q38_model_free(&m); rt_destroy(); return 1; }
serve_loop(&m);
rt_save(g_q38_usage, 0);
+ q38_prefix_cache_release(&m);
q38_model_free(&m); free_tokenizer(); rt_destroy();
return 0;
}
@@ -1559,6 +1841,16 @@ static int q38_u64_add(uint64_t *total,uint64_t value){
if(!total||*total>UINT64_MAX-value)return -1;*total+=value;return 0;
}
+static int q38_segment_tensor_storage_valid(const st_tensor *tensor){
+ uint64_t expected_bytes;
+ if(!tensor||tensor->dtype<0||tensor->dtype>6||tensor->numel<1||
+ tensor->nbytes<1||q38_u64_mul((uint64_t)tensor->numel,
+ (uint64_t)st_dtype_esz(tensor->dtype),
+ &expected_bytes)||
+ expected_bytes>INT64_MAX)return 0;
+ return tensor->nbytes==(int64_t)expected_bytes;
+}
+
enum {
Q38_EXPERT_FP8_BLOCK = 1u << 0,
Q38_EXPERT_FP8_EXPANDED = 1u << 1,
@@ -1575,10 +1867,15 @@ enum {
static int q38_segment_matrix_bytes(Model *m,const char *weight_name,
int rows,int cols,uint64_t *bytes,
unsigned *numeric_kinds){
- st_tensor *weight=st_find(&m->S,weight_name);uint64_t elements;
- if(!weight||weight->rank!=2||weight->shape[0]!=rows||
- weight->shape[1]!=cols||weight->numel!=(int64_t)rows*cols||
- q38_u64_mul((uint64_t)rows,(uint64_t)cols,&elements))return -1;
+ uint64_t elements;
+ if(!m||!weight_name||!bytes||!numeric_kinds||rows<1||cols<1||
+ q38_u64_mul((uint64_t)rows,(uint64_t)cols,&elements)||
+ elements>INT64_MAX)return -1;
+ st_tensor *weight=st_find(&m->S,weight_name);
+ if(!weight||!q38_segment_tensor_storage_valid(weight)||
+ (weight->dtype>2&&weight->dtype!=4)||weight->rank!=2||
+ weight->shape[0]!=rows||weight->shape[1]!=cols||
+ weight->numel!=(int64_t)elements)return -1;
if(weight->dtype==4){
char scale_name[340];
int length=snprintf(scale_name,sizeof scale_name,"%s_scale_inv",weight_name);
@@ -1587,7 +1884,8 @@ static int q38_segment_matrix_bytes(Model *m,const char *weight_name,
st_find(&m->S,scale_name):NULL;
if(!scale||scale->dtype<0||scale->dtype>2||scale->rank!=2||
scale->shape[0]!=block_rows||scale->shape[1]!=block_cols||
- scale->numel!=block_rows*block_cols)return -1;
+ scale->numel!=block_rows*block_cols||
+ !q38_segment_tensor_storage_valid(scale))return -1;
if(m->native_fp8){
uint64_t scale_bytes;
if(q38_u64_mul((uint64_t)scale->numel,sizeof(float),&scale_bytes)||
@@ -1611,7 +1909,9 @@ static int q38_segment_matrix_bytes(Model *m,const char *weight_name,
static int q38_segment_fused_matrix_bytes(Model *m,const st_tensor *tensor,
int64_t elements,uint64_t *bytes,
unsigned *numeric_kinds){
- if(!tensor||tensor->dtype<0||tensor->dtype>2||elements<1)return -1;
+ if(!m||!bytes||!numeric_kinds||!tensor||tensor->dtype<0||
+ tensor->dtype>2||elements<1||elements>tensor->numel||
+ !q38_segment_tensor_storage_valid(tensor))return -1;
uint64_t element_bytes=tensor->dtype==0&&m->native_bf16?
sizeof(uint16_t):sizeof(float);
if(q38_u64_mul((uint64_t)elements,element_bytes,bytes))return -1;
@@ -2022,3 +2322,274 @@ static int qwen38_segment_session_restore(void *impl,ColiSegmentReadFn rf,void *
static const ColiSegmentAdapter qwen38_segment_adapter={sizeof(ColiSegmentAdapter),COLI_SEGMENT_ABI_VERSION,"qwen38",qwen38_segment_engine_open,qwen38_segment_engine_destroy,qwen38_segment_session_create,qwen38_segment_session_destroy,qwen38_segment_session_run,qwen38_segment_session_snapshot,qwen38_segment_session_restore,{0}};
int coli_qwen38_segment_adapter_register(void){return coli_segment_adapter_register(&qwen38_segment_adapter);}
#endif /* COLI_SEGMENT_ADAPTER */
+
+#ifdef COLI_EDGE_ADAPTER
+/* ---------- engine-owned model Edge adapter --------------------------- */
+
+typedef struct {
+ Model model;
+} Qwen38EdgeEngine;
+
+/* Edge owns only the tokenizer and boundary tensors. Do not call
+ * q38_model_free(): a boundary-only model deliberately has no Layer/cache
+ * arrays, and keeping this destructor explicit makes it impossible for a
+ * later core change to make the Edge process load or free transformer state. */
+static void qwen38_edge_engine_destroy(void *engine_impl) {
+ Qwen38EdgeEngine *engine=(Qwen38EdgeEngine*)engine_impl;
+ if(!engine)return;
+ Model *model=&engine->model;
+ q38_weight_free(&model->embed);
+ q38_weight_free(&model->lm_head);
+ free(model->final_gr.norm);
+ q38_weight_free(&model->final_gr.down);
+ q38_weight_free(&model->final_gr.up);
+ free(model->c.is_attn);
+ st_destroy(&model->S);
+ free_tokenizer();
+ free(engine);
+}
+
+static int qwen38_edge_engine_open(
+ void **engine_impl,ColiEdgeCapabilities *capabilities,
+ const ColiEdgeEngineOptions *options,char *error,size_t error_size) {
+ if(!engine_impl||!capabilities||!options||!options->model_dir)
+ return coli_edge_adapter_error(error,error_size,
+ "invalid Qwen3.8 Edge open");
+ *engine_impl=NULL;
+ if(options->backend_mask&&(options->backend_mask&~COLI_EDGE_CAP_CPU))
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 Edge supports CPU only");
+ /* The production tokenizer is process-global. Segment-only engines do
+ * not touch it, so a distributed process may host every Segment adapter
+ * plus one Qwen3.8 Edge engine without cross-wiring vocabularies. */
+ if(g_tok)
+ return coli_edge_adapter_error(error,error_size,
+ "a Qwen3.8 tokenizer is already active");
+ Qwen38EdgeEngine *engine=(Qwen38EdgeEngine*)calloc(1,sizeof(*engine));
+ if(!engine)
+ return coli_edge_adapter_error(error,error_size,
+ "out of memory opening Qwen3.8 Edge");
+ Model *model=&engine->model;
+ model->native_bf16=q38_env_bool("Q38_NATIVE_BF16",1);
+ model->native_fp8=q38_env_bool("Q38_NATIVE_FP8",1);
+ q38_load_cfg(&model->c,options->model_dir);
+ q38_validate_cfg(&model->c);
+ st_init(&model->S,options->model_dir);
+ if(st_has(&model->S,"model.language_model.embed_tokens.weight"))
+ snprintf(model->prefix,sizeof model->prefix,"model.language_model");
+ else if(st_has(&model->S,"model.embed_tokens.weight"))
+ snprintf(model->prefix,sizeof model->prefix,"model");
+ else {
+ qwen38_edge_engine_destroy(engine);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 checkpoint has no text embedding");
+ }
+
+ Cfg *config=&model->c;char name[320];
+ snprintf(name,sizeof name,"%s.embed_tokens.weight",model->prefix);
+ model->embed=q38_load_weight(model,name,config->vocab,config->hidden);
+ model->lm_head=q38_load_weight(model,"lm_head.weight",
+ config->vocab,config->hidden);
+ q38_load_gr(model,&model->final_gr,-1,NULL,0);
+
+ char tokenizer_path[4096];
+ int path_length=snprintf(tokenizer_path,sizeof tokenizer_path,
+ "%s/tokenizer.json",options->model_dir);
+ if(path_length<0||(size_t)path_length>=sizeof tokenizer_path||
+ load_tokenizer(tokenizer_path)) {
+ qwen38_edge_engine_destroy(engine);
+ return coli_edge_adapter_error(error,error_size,
+ "cannot load Qwen3.8 tokenizer");
+ }
+
+ uint64_t resident=model->resident_weight_bytes;
+ uint64_t norm_bytes=(uint64_t)config->hc_width*sizeof(float);
+ if(resident>UINT64_MAX-norm_bytes) {
+ qwen38_edge_engine_destroy(engine);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 Edge resident size overflows");
+ }
+ resident+=norm_bytes;
+ if(options->memory_limit_bytes&&resident>options->memory_limit_bytes) {
+ qwen38_edge_engine_destroy(engine);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 Edge exceeds memory limit");
+ }
+
+ /* Numeric compatibility includes the actual routed-expert representation
+ * even though Edge does not load those tensors. This must be byte-for-byte
+ * identical to the full Segment range or the runtime refuses the pairing. */
+ uint64_t slot_bytes=0,fixed_scale_bytes=0;unsigned numeric_kinds=0;
+ if(q38_segment_expert_layout(model,0,(uint32_t)config->layers,
+ &slot_bytes,&fixed_scale_bytes,
+ &numeric_kinds)) {
+ qwen38_edge_engine_destroy(engine);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 Edge checkpoint has an unsupported expert layout");
+ }
+ (void)slot_bytes;(void)fixed_scale_bytes;
+
+ memset(capabilities,0,sizeof(*capabilities));
+ capabilities->struct_size=sizeof(*capabilities);
+ capabilities->abi_version=COLI_EDGE_ABI_VERSION;
+ capabilities->flags=COLI_EDGE_CAP_TOKENIZE|COLI_EDGE_CAP_DETOKENIZE|
+ COLI_EDGE_CAP_GREEDY|COLI_EDGE_CAP_CPU;
+ coli_edge_capability_string(capabilities->engine_id,
+ sizeof(capabilities->engine_id),"qwen38");
+ coli_edge_capability_string(
+ capabilities->state_schema,sizeof(capabilities->state_schema),
+ "qwen38/hyper-kv-deltanet-ple-f32-v1");
+ q38_segment_numeric_class(capabilities->numeric_class,
+ sizeof(capabilities->numeric_class),numeric_kinds);
+ coli_edge_capability_string(
+ capabilities->tokenizer_class,sizeof(capabilities->tokenizer_class),
+ "qwen38/hf-byte-bpe-nfc-v1");
+ capabilities->state_dtype=COLI_EDGE_DTYPE_F32;
+ capabilities->state_width=(uint32_t)config->hc_width;
+ capabilities->vocab_size=(uint32_t)config->vocab;
+ capabilities->max_batch_rows=128;
+ capabilities->max_context_tokens=(uint32_t)config->max_positions;
+ capabilities->num_layers=(uint32_t)config->layers;
+ capabilities->bos_token_id=-1;
+ capabilities->eos_token_id=config->eos_id;
+ capabilities->resident_bytes=resident;
+ *engine_impl=engine;
+ return 0;
+}
+
+static int qwen38_edge_tokenize(
+ void *engine_impl,const char *text,size_t text_bytes,
+ int32_t *token_ids,size_t token_capacity,size_t *token_count,
+ char *error,size_t error_size) {
+ (void)engine_impl;
+ if(!text||!token_count||text_bytes>INT_MAX)
+ return coli_edge_adapter_error(error,error_size,
+ "invalid Qwen3.8 tokenizer input");
+ int *ids=NULL,count=0;
+ encode_text_n(text,text_bytes,&ids,&count);
+ if(count<0||(token_ids&&token_capacity<(size_t)count)) {
+ free(ids);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 token output buffer is too small");
+ }
+ *token_count=(size_t)count;
+ if(token_ids)
+ for(int item=0;item<count;item++)token_ids[item]=ids[item];
+ free(ids);
+ return 0;
+}
+
+static int qwen38_edge_detokenize(
+ void *engine_impl,const int32_t *token_ids,size_t token_count,
+ char *text,size_t text_capacity,size_t *text_bytes,
+ char *error,size_t error_size) {
+ (void)engine_impl;
+ if(!token_ids||!token_count||!text_bytes||token_count>INT_MAX||
+ token_count>SIZE_MAX/sizeof(int))
+ return coli_edge_adapter_error(error,error_size,
+ "invalid Qwen3.8 detokenizer input");
+ int *ids=(int*)malloc(token_count*sizeof(*ids));
+ if(!ids)
+ return coli_edge_adapter_error(error,error_size,
+ "out of memory detokenizing Qwen3.8 tokens");
+ for(size_t item=0;item<token_count;item++)ids[item]=token_ids[item];
+ unsigned char *decoded=NULL;size_t decoded_bytes=0;
+ int result=decode_range_alloc(ids,0,(int)token_count,
+ &decoded,&decoded_bytes);
+ free(ids);
+ if(result) {
+ free(decoded);
+ return coli_edge_adapter_error(error,error_size,
+ "cannot detokenize Qwen3.8 tokens");
+ }
+ *text_bytes=decoded_bytes;
+ if(text&&text_capacity<decoded_bytes+1u) {
+ free(decoded);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 text output buffer is too small");
+ }
+ if(text)memcpy(text,decoded,decoded_bytes+1u);
+ free(decoded);
+ return 0;
+}
+
+static int qwen38_edge_embed(void *engine_impl,
+ const ColiEdgeEmbedRequest *request,
+ char *error,size_t error_size) {
+ Qwen38EdgeEngine *engine=(Qwen38EdgeEngine*)engine_impl;
+ Cfg *config=&engine->model.c;
+ float *output=(float*)request->output;
+ float *row=(float*)malloc((size_t)config->hidden*sizeof(*row));
+ if(!row)
+ return coli_edge_adapter_error(error,error_size,
+ "out of memory embedding Qwen3.8 tokens");
+ for(uint32_t index=0;index<request->rows;index++) {
+ if(request->should_cancel&&
+ request->should_cancel(request->cancel_user_data)) {
+ free(row);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 Edge embedding cancelled");
+ }
+ int token=request->token_ids[index];
+ if(token<0||token>=config->vocab) {
+ free(row);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 token ID is out of range");
+ }
+ q38_weight_row(&engine->model.embed,token,row);
+ float *state=output+(size_t)index*config->hc_width;
+ for(int copy=0;copy<config->hc_count;copy++)
+ memcpy(state+(size_t)copy*config->hidden,row,
+ (size_t)config->hidden*sizeof(*row));
+ }
+ free(row);
+ return 0;
+}
+
+static int qwen38_edge_select(void *engine_impl,
+ const ColiEdgeSelectRequest *request,
+ char *error,size_t error_size) {
+ Qwen38EdgeEngine *engine=(Qwen38EdgeEngine*)engine_impl;
+ Cfg *config=&engine->model.c;
+ float *hidden=(float*)malloc((size_t)config->hidden*sizeof(*hidden));
+ float *logits=(float*)malloc((size_t)config->vocab*sizeof(*logits));
+ if(!hidden||!logits) {
+ free(logits);free(hidden);
+ return coli_edge_adapter_error(error,error_size,
+ "out of memory running Qwen3.8 head");
+ }
+ const float *input=(const float*)request->input;
+ for(uint32_t row=0;row<request->rows;row++) {
+ if(request->should_cancel&&
+ request->should_cancel(request->cancel_user_data)) {
+ free(logits);free(hidden);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 Edge selection cancelled");
+ }
+ q38_gr_read(&engine->model,&engine->model.final_gr,
+ input+(size_t)row*config->hc_width,1,hidden,NULL);
+ q38_weight_matmul(logits,hidden,&engine->model.lm_head,
+ 1,config->hidden,config->vocab);
+ if(coli_edge_argmax(logits,(uint32_t)config->vocab,
+ &request->token_ids[row],
+ request->scores?&request->scores[row]:NULL)) {
+ free(logits);free(hidden);
+ return coli_edge_adapter_error(error,error_size,
+ "Qwen3.8 Edge head failed");
+ }
+ }
+ free(logits);free(hidden);
+ return 0;
+}
+
+static const ColiEdgeAdapter qwen38_edge_adapter={
+ sizeof(ColiEdgeAdapter),COLI_EDGE_ABI_VERSION,"qwen38",
+ qwen38_edge_engine_open,qwen38_edge_engine_destroy,
+ qwen38_edge_tokenize,qwen38_edge_detokenize,
+ qwen38_edge_embed,qwen38_edge_select,{0}
+};
+
+int coli_qwen38_edge_adapter_register(void) {
+ return coli_edge_adapter_register(&qwen38_edge_adapter);
+}
+#endif /* COLI_EDGE_ADAPTER */
diff --git a/c/qwen38_core.h b/c/qwen38_core.h
index 67e09c61..3a594fe1 100644
--- a/c/qwen38_core.h
+++ b/c/qwen38_core.h
@@ -13,6 +13,8 @@
#define Q38_MAX_EXPERTS 1024
#define Q38_MAX_TOPK 256
#define Q38_MAX_PLE_PARTS 512
+#define Q38_PREFILL_BATCH_ROWS 32
+#define Q38_PREFILL_WORKSPACE_BYTES (64u << 20)
typedef struct {
int hidden, layers, vocab, max_positions, eos_id;
@@ -127,6 +129,7 @@ typedef struct {
int ple_history_len;
int range_begin, range_end;
int native_fp8, native_bf16, expert_prefetch, expert_parallel_reads;
+ int prefill_batch;
uint64_t resident_weight_bytes;
double dense_load_s;
Q38Timers timers;
@@ -646,6 +649,7 @@ static void model_init_range(Model *m,const char *snap,int cap,int bits,
m->native_bf16=q38_env_bool("Q38_NATIVE_BF16",1);
m->expert_prefetch=q38_env_bool("Q38_EXPERT_PREFETCH",1);
m->expert_parallel_reads=q38_env_bool("Q38_EXPERT_PARALLEL_READS",1);
+ m->prefill_batch=q38_env_bool("Q38_PREFILL_BATCH",1);
q38_load_cfg(&m->c,snap); q38_validate_cfg(&m->c); st_init(&m->S,snap);
Cfg *c=&m->c; char nm[320];
if(st_has(&m->S,"model.language_model.embed_tokens.weight")) snprintf(m->prefix,sizeof m->prefix,"model.language_model");
@@ -1207,45 +1211,126 @@ static void q38_ple(Model *m,const int *ids,int S,const float *hyper,float *out)
q38_tm_add(m,Q38_TM_PLE,phase_started);
}
-static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,float *out) {
+/* Choose a context-independent prefill chunk whose private workspace fits the
+ * common target. Callers provide exact fixed and per-row byte counts; even a
+ * hostile-but-valid geometry gets one row rather than an unbounded allocation. */
+static int q38_bounded_prefill_rows(int requested,uint64_t fixed,
+ uint64_t per_row) {
+ int rows=requested<Q38_PREFILL_BATCH_ROWS?requested:Q38_PREFILL_BATCH_ROWS;
+ if(rows<1)return 1;
+ for(;rows>1;rows--)
+ if(per_row<=UINT64_MAX/(uint64_t)rows&&
+ fixed<=UINT64_MAX-per_row*(uint64_t)rows&&
+ fixed+per_row*(uint64_t)rows<=Q38_PREFILL_WORKSPACE_BYTES)
+ return rows;
+ return 1;
+}
+
+/* Batch the four resident DeltaNet input projections and the output projection
+ * in bounded chunks. The convolution and recurrent update remain strictly
+ * token-causal inside each chunk, so chunk boundaries cannot change state or
+ * floating-point order. */
+static void q38_deltanet(Model *m,Layer *l,int layer,const float *x,int S,
+ float *out) {
double phase_started=now_s();
- Cfg *c=&m->c; int H=c->hidden,VH=c->dn_vheads,KH=c->dn_kheads,KD=c->dn_kdim,VD=c->dn_vdim;
- int CD=c->dn_conv_dim,CK=c->dn_convk,V=VH*VD,K=KH*KD,rep=VH/KH;
- float *qkv=falloc(CD),*z=falloc(V),*bb=falloc(VH),*aa=falloc(VH),*conv=falloc(CD);
- float *q=falloc((int64_t)VH*KD),*k=falloc((int64_t)VH*KD),*core=falloc(V),*norm=falloc(V);
+ Cfg *c=&m->c;
+ int H=c->hidden,VH=c->dn_vheads,KH=c->dn_kheads;
+ int KD=c->dn_kdim,VD=c->dn_vdim,CD=c->dn_conv_dim,CK=c->dn_convk;
+ int V=VH*VD,K=KH*KD,rep=VH/KH;
+ uint64_t row_floats=(uint64_t)CD+2u*(uint64_t)V+2u*(uint64_t)VH;
+ uint64_t fixed_floats=(uint64_t)CD+2u*(uint64_t)VH*(uint64_t)KD+
+ (uint64_t)V;
+ uint64_t row_bytes=row_floats>UINT64_MAX/sizeof(float)?
+ UINT64_MAX:row_floats*sizeof(float);
+ uint64_t fixed_bytes=fixed_floats>UINT64_MAX/sizeof(float)?
+ UINT64_MAX:fixed_floats*sizeof(float);
+ int rows_capacity=m->prefill_batch?
+ q38_bounded_prefill_rows(S,fixed_bytes,row_bytes):1;
+
+ float *qkv=falloc((int64_t)rows_capacity*CD);
+ float *z=falloc((int64_t)rows_capacity*V);
+ float *bb=falloc((int64_t)rows_capacity*VH);
+ float *aa=falloc((int64_t)rows_capacity*VH);
+ float *norm=falloc((int64_t)rows_capacity*V);
+ float *conv=falloc(CD);
+ float *q=falloc((int64_t)VH*KD),*k=falloc((int64_t)VH*KD);
+ float *core=falloc(V);
float *rec=m->DN_rec[layer],*ring=m->DN_conv[layer];
- for(int s=0;s<S;s++){
- const float *xs=x+(int64_t)s*H;
- q38_dense_matmul(m,qkv,xs,&l->dn_qkv,1,H,CD);q38_dense_matmul(m,z,xs,&l->dn_z,1,H,V);
- q38_dense_matmul(m,bb,xs,&l->dn_b,1,H,VH);q38_dense_matmul(m,aa,xs,&l->dn_a,1,H,VH);
- for(int d=0;d<CD;d++){
- float a=l->dn_conv[(int64_t)d*CK+CK-1]*qkv[d];float *r=ring+(int64_t)d*(CK-1);
- for(int j=0;j<CK-1;j++)a+=l->dn_conv[(int64_t)d*CK+j]*r[j];conv[d]=q38_silu(a);
- for(int j=0;j<CK-2;j++)r[j]=r[j+1];r[CK-2]=qkv[d];
- }
- const float *qi=conv,*ki=conv+K,*vi=conv+2*K;
- for(int h=0;h<VH;h++){
- memcpy(q+(int64_t)h*KD,qi+(int64_t)(h/rep)*KD,(size_t)KD*sizeof(float));
- memcpy(k+(int64_t)h*KD,ki+(int64_t)(h/rep)*KD,(size_t)KD*sizeof(float));
- double qs=1e-6,ks=1e-6;for(int d=0;d<KD;d++){qs+=(double)q[(int64_t)h*KD+d]*q[(int64_t)h*KD+d];ks+=(double)k[(int64_t)h*KD+d]*k[(int64_t)h*KD+d];}
- float qr=1.f/sqrtf((float)qs)/sqrtf((float)KD),kr=1.f/sqrtf((float)ks);
- for(int d=0;d<KD;d++){q[(int64_t)h*KD+d]*=qr;k[(int64_t)h*KD+d]*=kr;}
- }
- #pragma omp parallel for schedule(static)
- for(int h=0;h<VH;h++){
- float *state=rec+(int64_t)h*KD*VD;const float *qh=q+(int64_t)h*KD,*kh=k+(int64_t)h*KD,*vh=vi+(int64_t)h*VD;
- float alpha=expf(-expf(l->dn_alog[h])*q38_softplus(aa[h]+l->dn_dtbias[h]));
- float beta=q38_sigmoid(bb[h]);float delta[512];
- int64_t state_cells=(int64_t)KD*VD;
- for(int64_t z0=0;z0<state_cells;z0++)state[z0]*=alpha;
- for(int v0=0;v0<VD;v0++){float a=0.f;for(int d=0;d<KD;d++)a+=kh[d]*state[(int64_t)d*VD+v0];delta[v0]=(vh[v0]-a)*beta;}
- for(int d=0;d<KD;d++)for(int v0=0;v0<VD;v0++)state[(int64_t)d*VD+v0]+=kh[d]*delta[v0];
- for(int v0=0;v0<VD;v0++){float a=0.f;for(int d=0;d<KD;d++)a+=qh[d]*state[(int64_t)d*VD+v0];core[(int64_t)h*VD+v0]=a;}
+
+ for(int base=0;base<S;) {
+ int rows=S-base<rows_capacity?S-base:rows_capacity;
+ const float *chunk=x+(int64_t)base*H;
+ q38_dense_matmul(m,qkv,chunk,&l->dn_qkv,rows,H,CD);
+ q38_dense_matmul(m,z,chunk,&l->dn_z,rows,H,V);
+ q38_dense_matmul(m,bb,chunk,&l->dn_b,rows,H,VH);
+ q38_dense_matmul(m,aa,chunk,&l->dn_a,rows,H,VH);
+
+ for(int s=0;s<rows;s++) {
+ float *qkv_row=qkv+(int64_t)s*CD;
+ float *z_row=z+(int64_t)s*V;
+ float *b_row=bb+(int64_t)s*VH;
+ float *a_row=aa+(int64_t)s*VH;
+ for(int d=0;d<CD;d++) {
+ float value=l->dn_conv[(int64_t)d*CK+CK-1]*qkv_row[d];
+ float *history=ring+(int64_t)d*(CK-1);
+ for(int tap=0;tap<CK-1;tap++)
+ value+=l->dn_conv[(int64_t)d*CK+tap]*history[tap];
+ conv[d]=q38_silu(value);
+ for(int tap=0;tap<CK-2;tap++)history[tap]=history[tap+1];
+ history[CK-2]=qkv_row[d];
+ }
+ const float *qi=conv,*ki=conv+K,*vi=conv+2*K;
+ for(int h=0;h<VH;h++) {
+ float *qh=q+(int64_t)h*KD,*kh=k+(int64_t)h*KD;
+ memcpy(qh,qi+(int64_t)(h/rep)*KD,(size_t)KD*sizeof(float));
+ memcpy(kh,ki+(int64_t)(h/rep)*KD,(size_t)KD*sizeof(float));
+ double qsum=1e-6,ksum=1e-6;
+ for(int d=0;d<KD;d++) {
+ qsum+=(double)qh[d]*qh[d];
+ ksum+=(double)kh[d]*kh[d];
+ }
+ float qscale=1.f/sqrtf((float)qsum)/sqrtf((float)KD);
+ float kscale=1.f/sqrtf((float)ksum);
+ for(int d=0;d<KD;d++){qh[d]*=qscale;kh[d]*=kscale;}
+ }
+ #pragma omp parallel for schedule(static)
+ for(int h=0;h<VH;h++) {
+ float *state=rec+(int64_t)h*KD*VD;
+ const float *qh=q+(int64_t)h*KD;
+ const float *kh=k+(int64_t)h*KD;
+ const float *vh=vi+(int64_t)h*VD;
+ float alpha=expf(-expf(l->dn_alog[h])*
+ q38_softplus(a_row[h]+l->dn_dtbias[h]));
+ float beta=q38_sigmoid(b_row[h]);
+ float delta[512];
+ int64_t state_cells=(int64_t)KD*VD;
+ for(int64_t cell=0;cell<state_cells;cell++)state[cell]*=alpha;
+ for(int value=0;value<VD;value++) {
+ float previous=0.f;
+ for(int d=0;d<KD;d++)
+ previous+=kh[d]*state[(int64_t)d*VD+value];
+ delta[value]=(vh[value]-previous)*beta;
+ }
+ for(int d=0;d<KD;d++)for(int value=0;value<VD;value++)
+ state[(int64_t)d*VD+value]+=kh[d]*delta[value];
+ for(int value=0;value<VD;value++) {
+ float current=0.f;
+ for(int d=0;d<KD;d++)
+ current+=qh[d]*state[(int64_t)d*VD+value];
+ core[(int64_t)h*VD+value]=current;
+ }
+ }
+ float *norm_row=norm+(int64_t)s*V;
+ for(int h=0;h<VH;h++)
+ q38_rmsg(norm_row+(int64_t)h*VD,core+(int64_t)h*VD,
+ z_row+(int64_t)h*VD,l->dn_norm,VD,c->eps,1);
}
- for(int h=0;h<VH;h++)q38_rmsg(norm+(int64_t)h*VD,core+(int64_t)h*VD,z+(int64_t)h*VD,l->dn_norm,VD,c->eps,1);
- q38_dense_matmul(m,out+(int64_t)s*H,norm,&l->dn_out,1,V,H);
+ q38_dense_matmul(m,out+(int64_t)base*H,norm,&l->dn_out,
+ rows,V,H);
+ base+=rows;
}
- free(qkv);free(z);free(bb);free(aa);free(conv);free(q);free(k);free(core);free(norm);
+ free(qkv);free(z);free(bb);free(aa);free(norm);free(conv);
+ free(q);free(k);free(core);
q38_tm_add(m,Q38_TM_DELTANET,phase_started);
}
@@ -1306,7 +1391,10 @@ static void q38_attention(Model *m,Layer *l,int layer,const float *x,int S,int p
free(qp);free(kp);free(vp);free(ip);free(heads);free(qidx);free(pool);free(selected);
}
-static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out) {
+/* The single-row path is intentionally kept separate from prefill. Decode is
+ * the latency-sensitive steady state and should not pay for route tables or a
+ * prompt-sized workspace. */
+static void q38_moe_decode(Model *m,Layer *l,int layer,const float *x,int S,float *out) {
Cfg *c=&m->c;int H=c->hidden,E=c->experts,K=c->topk,I=c->inter,SI=c->shared_inter;
float *logits=falloc(E),*sg=falloc(SI),*su=falloc(SI),*sh=falloc(SI),*shared=falloc(H);
float *eg=falloc(I),*eu=falloc(I),*eh=falloc(I),*eo=falloc(H);
@@ -1341,6 +1429,228 @@ static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out)
free(logits);free(sg);free(su);free(sh);free(shared);free(eg);free(eu);free(eh);free(eo);
}
+typedef struct {
+ int expert;
+ float gate;
+} Q38RouteAssignment;
+
+/* Pick a prefill size from a fixed workspace budget. The number of rows is
+ * deliberately bounded independently of the context length: a long prompt
+ * therefore reuses the same route, expert and matmul buffers one chunk at a
+ * time. A single assignment needs input, gate/up activations and output;
+ * these are the only buffers that scale with the number of routed rows. */
+static int q38_moe_prefill_rows(const Cfg *c,int requested) {
+ int64_t H=c->hidden,I=c->inter,E=c->experts,K=c->topk,SI=c->shared_inter;
+ uint64_t per_assignment=(uint64_t)(2LL*((int64_t)H+I))*sizeof(float)+
+ sizeof(Q38RouteAssignment)+2*sizeof(int);
+ uint64_t per_row=(uint64_t)E*sizeof(float)+
+ (uint64_t)(3LL*(int64_t)SI+H+1)*sizeof(float);
+ uint64_t fixed=(uint64_t)E*(4*sizeof(int)+2*sizeof(Slot*));
+ if(fixed<=UINT64_MAX-sizeof(int))fixed+=sizeof(int); /* group_offsets[E] */
+ else fixed=UINT64_MAX;
+ uint64_t assignment_row=(uint64_t)K>UINT64_MAX/per_assignment?
+ UINT64_MAX:(uint64_t)K*per_assignment;
+ uint64_t total_row=per_row>UINT64_MAX-assignment_row?
+ UINT64_MAX:per_row+assignment_row;
+ return q38_bounded_prefill_rows(requested,fixed,total_row);
+}
+
+/* Prefill MoE: route a bounded row chunk first, then execute each distinct
+ * expert's assignments as one batched SwiGLU. Grouping is an I/O optimization
+ * only. Expert outputs are placed back in assignment order and the final
+ * weighted reduction still visits rank 0..top-k-1 for every row, preserving the
+ * decode path's floating-point accumulation order. */
+static void q38_moe_prefill(Model *m,Layer *l,int layer,const float *x,
+ int S,float *out) {
+ Cfg *c=&m->c;
+ int H=c->hidden,E=c->experts,K=c->topk,I=c->inter,SI=c->shared_inter;
+ int rows_capacity=q38_moe_prefill_rows(c,S);
+ int64_t max_assign=(int64_t)rows_capacity*K;
+
+ Q38RouteAssignment *routes=(Q38RouteAssignment*)malloc(
+ (size_t)max_assign*sizeof(*routes));
+ int *assignments=(int*)malloc((size_t)max_assign*sizeof(*assignments));
+ int *assignment_positions=(int*)malloc((size_t)max_assign*sizeof(*assignment_positions));
+ int *group_counts=(int*)calloc((size_t)E,sizeof(*group_counts));
+ int *group_offsets=(int*)malloc((size_t)(E+1)*sizeof(*group_offsets));
+ int *group_cursor=(int*)malloc((size_t)E*sizeof(*group_cursor));
+ int *unique=(int*)malloc((size_t)E*sizeof(*unique));
+ Slot **batch_slots=(Slot**)calloc((size_t)E,sizeof(*batch_slots));
+ if(!routes||!assignments||!assignment_positions||!group_counts||
+ !group_offsets||!group_cursor||!unique||!batch_slots){
+ fprintf(stderr,"OOM Qwen3.8 MoE prefill metadata\n");exit(1);
+ }
+
+ float *logits=falloc((int64_t)rows_capacity*E);
+ float *shared_gate=falloc(rows_capacity);
+ float *shared_g=falloc((int64_t)rows_capacity*SI);
+ float *shared_u=falloc((int64_t)rows_capacity*SI);
+ float *shared_hidden=falloc((int64_t)rows_capacity*SI);
+ float *shared_out=falloc((int64_t)rows_capacity*H);
+ float *expert_input=falloc(max_assign*H);
+ float *expert_gate=falloc(max_assign*I);
+ float *expert_up=falloc(max_assign*I);
+ float *routed_out=falloc(max_assign*H);
+
+ for(int base=0;base<S;) {
+ int rows=S-base<rows_capacity?S-base:rows_capacity;
+ int assignment_count=rows*K;
+ memset(group_counts,0,(size_t)E*sizeof(*group_counts));
+
+ /* Route the complete chunk with one resident router matmul. Selection
+ * intentionally mirrors q38_moe_decode, including rt_router_pick's
+ * deterministic fallback for invalid logits. */
+ q38_dense_matmul(m,logits,x+(int64_t)base*H,&l->router,rows,H,E);
+ for(int s=0;s<rows;s++) {
+ float *probabilities=logits+(int64_t)s*E;
+ float maximum=probabilities[0];
+ for(int e=1;e<E;e++)if(probabilities[e]>maximum)maximum=probabilities[e];
+ double total=0.0;
+ for(int e=0;e<E;e++) {
+ probabilities[e]=expf(probabilities[e]-maximum);
+ total+=probabilities[e];
+ }
+ int selected[Q38_MAX_TOPK];
+ float selected_probability[Q38_MAX_TOPK];
+ for(int rank=0;rank<K;rank++) {
+ int best=-1;float best_value=-1.f;
+ for(int e=0;e<E;e++) {
+ int used=0;
+ for(int previous=0;previous<rank;previous++)
+ if(selected[previous]==e){used=1;break;}
+ if(!used&&probabilities[e]>best_value){
+ best_value=probabilities[e];best=e;
+ }
+ }
+ selected[rank]=rt_router_pick(best,rank,E,layer);
+ selected_probability[rank]=probabilities[selected[rank]];
+ }
+ double top=0.0;
+ for(int rank=0;rank<K;rank++)top+=selected_probability[rank];
+ double denominator=c->norm_topk?top:total;
+ float gates[Q38_MAX_TOPK];
+ for(int rank=0;rank<K;rank++) {
+ gates[rank]=(float)(selected_probability[rank]/denominator);
+ int assignment=s*K+rank;
+ routes[assignment]=(Q38RouteAssignment){selected[rank],gates[rank]};
+ group_counts[selected[rank]]++;
+ }
+ rt_route(layer,base+s,selected,gates,K);
+ }
+
+ /* Prefixing by expert makes every group contiguous while the inverse
+ * map lets the final reduction recover the original row/rank order. */
+ group_offsets[0]=0;
+ int unique_count=0;
+ for(int e=0;e<E;e++) {
+ group_offsets[e+1]=group_offsets[e]+group_counts[e];
+ if(group_counts[e])unique[unique_count++]=e;
+ }
+ memset(group_cursor,0,(size_t)E*sizeof(*group_cursor));
+ for(int assignment=0;assignment<assignment_count;assignment++) {
+ int expert=routes[assignment].expert;
+ int position=group_offsets[expert]+group_cursor[expert]++;
+ assignments[position]=assignment;
+ assignment_positions[assignment]=position;
+ }
+
+ /* One advice range per distinct expert is enough for this chunk. */
+ q38_prefetch_native_fp8_experts(m,layer,unique,unique_count);
+
+ /* Shared expert work is independent across rows and remains resident;
+ * batching it here also keeps its cost out of the routed groups. */
+ double phase_started=now_s();
+ q38_weight_matmul(shared_g,x+(int64_t)base*H,&l->sh_g,rows,H,SI);
+ q38_weight_matmul(shared_u,x+(int64_t)base*H,&l->sh_u,rows,H,SI);
+ for(int s=0;s<rows;s++)
+ for(int j=0;j<SI;j++)
+ shared_hidden[(int64_t)s*SI+j]=
+ q38_silu(shared_g[(int64_t)s*SI+j])*
+ shared_u[(int64_t)s*SI+j];
+ q38_weight_matmul(shared_out,shared_hidden,&l->sh_d,rows,SI,H);
+ for(int s=0;s<rows;s++) {
+ const float *xs=x+(int64_t)(base+s)*H;
+ float gate=0.f;
+ for(int d=0;d<H;d++)gate+=xs[d]*l->sh_gate[d];
+ shared_gate[s]=q38_sigmoid(gate);
+ }
+ q38_tm_add(m,Q38_TM_SHARED_EXPERT,phase_started);
+
+ /* A demand-set batch is particularly effective for native FP8: reserve
+ * all slots before workers read the two coalesced ranges. Other native
+ * layouts use the same grouped matmul loop with the ordinary bounded
+ * LRU loader, so no format loses correctness or caching. */
+ memset(out+(int64_t)base*H,0,(size_t)rows*H*sizeof(float));
+ /* A prompt chunk can route to more distinct experts than the retained
+ * cache can hold. Load and consume cache-sized groups instead of
+ * declining the complete parallel demand set: every expert is still
+ * loaded once for this chunk, and a later group may safely reuse its
+ * slots because the preceding outputs already live in routed_out. */
+ int load_limit=m->cache[layer].cap;
+ if(load_limit>Q38_MAX_TOPK)load_limit=Q38_MAX_TOPK;
+ if(load_limit<1)load_limit=1;
+ for(int unique_base=0;unique_base<unique_count;) {
+ int load_count=unique_count-unique_base;
+ if(load_count>load_limit)load_count=load_limit;
+ int loaded_batch=load_count>=2&&q38_expert_get_batch(
+ m,layer,unique+unique_base,load_count,batch_slots);
+ for(int offset=0;offset<load_count;offset++) {
+ int e=unique[unique_base+offset];
+ int count=group_counts[e];
+ Slot *expert=loaded_batch?batch_slots[offset]:
+ q38_expert_get(m,layer,e);
+ int first=group_offsets[e];
+ for(int a=0;a<count;a++) {
+ int assignment=assignments[first+a];
+ int row=assignment/K;
+ memcpy(expert_input+(int64_t)a*H,
+ x+(int64_t)(base+row)*H,(size_t)H*sizeof(float));
+ }
+ phase_started=now_s();
+ q38_weight_matmul(expert_gate,expert_input,&expert->gate,
+ count,H,I);
+ q38_weight_matmul(expert_up,expert_input,&expert->up,count,H,I);
+ for(int a=0;a<count;a++)for(int j=0;j<I;j++)
+ expert_gate[(int64_t)a*I+j]=
+ q38_silu(expert_gate[(int64_t)a*I+j])*
+ expert_up[(int64_t)a*I+j];
+ q38_weight_matmul(routed_out+(int64_t)first*H,expert_gate,
+ &expert->down,count,I,H);
+ q38_tm_add(m,Q38_TM_ROUTED_EXPERT,phase_started);
+ }
+ unique_base+=load_count;
+ }
+
+ /* The grouped execution above is intentionally not the reduction
+ * order. Replaying the original top-k sequence gives the same result
+ * as the single-row implementation for F32, BF16 and block-FP8. */
+ for(int s=0;s<rows;s++) {
+ float *ys=out+(int64_t)(base+s)*H;
+ for(int rank=0;rank<K;rank++) {
+ int assignment=s*K+rank;
+ const float *expert_output=routed_out+
+ (int64_t)assignment_positions[assignment]*H;
+ float gate=routes[assignment].gate;
+ for(int d=0;d<H;d++)ys[d]+=gate*expert_output[d];
+ }
+ const float *shared=shared_out+(int64_t)s*H;
+ for(int d=0;d<H;d++)ys[d]+=shared_gate[s]*shared[d];
+ }
+ base+=rows;
+ }
+ rt_trace_end();
+ free(logits);free(shared_gate);free(shared_g);free(shared_u);
+ free(shared_hidden);free(shared_out);free(expert_input);free(expert_gate);
+ free(expert_up);free(routed_out);free(routes);free(assignments);
+ free(assignment_positions);free(group_counts);free(group_offsets);
+ free(group_cursor);free(unique);free(batch_slots);
+}
+
+static void q38_moe(Model *m,Layer *l,int layer,const float *x,int S,float *out) {
+ if(S<=1||!m->prefill_batch)q38_moe_decode(m,l,layer,x,S,out);
+ else q38_moe_prefill(m,l,layer,x,S,out);
+}
+
static void reset_recurrent(Model *m) {
Cfg *c=&m->c;
for(int i=0;i<c->layers;i++)if(!c->is_attn[i]){
@@ -1463,13 +1773,15 @@ static void q38_layer_free(Layer *l) {
static void q38_model_free(Model *m) {
if(!m) return;
for(int i=0;i<m->c.layers;i++) {
- q38_layer_free(&m->L[i]);
+ if(m->L)q38_layer_free(&m->L[i]);
if(m->cache) {
- for(int s=0;s<m->cache[i].n;s++){
- q38_weight_free(&m->cache[i].slots[s].gate);
- q38_weight_free(&m->cache[i].slots[s].up);
- q38_weight_free(&m->cache[i].slots[s].down);
- free(m->cache[i].slots[s].fp8_slab);
+ if(m->cache[i].slots) {
+ for(int s=0;s<m->cache[i].n;s++) {
+ q38_weight_free(&m->cache[i].slots[s].gate);
+ q38_weight_free(&m->cache[i].slots[s].up);
+ q38_weight_free(&m->cache[i].slots[s].down);
+ free(m->cache[i].slots[s].fp8_slab);
+ }
}
free(m->cache[i].slots); free(m->cache[i].by_expert);
}
diff --git a/c/resource_plan.py b/c/resource_plan.py
index f671f4ee..c80b1cbe 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -12,7 +12,8 @@ import threading
from pathlib import Path
from family_registry import (expert_contributions, planner_geometry,
- resident_contribution, resolve_model)
+ fixed_resident_contribution, resident_contribution,
+ resolve_model)
GB = 1_000_000_000
@@ -24,7 +25,7 @@ EXPERT_RE = re.compile(r"(?:model\.)?layers\.(\d+)\.(?:mlp|ffn)\.experts\.(\d+)\
# sidecar and self-invalidate on any change. Best-effort: any read/write failure falls
# straight back to a full recompute (see analyze_model). Sits alongside .coli_usage/.coli_ssd.
_ANALYSIS_CACHE_NAME = ".coli_analysis.json"
-_ANALYSIS_CACHE_VERSION = 6
+_ANALYSIS_CACHE_VERSION = 7
def _analysis_signature(shards, config_path):
@@ -99,6 +100,10 @@ def analyze_model(model):
pass # missing/corrupt/unreadable cache -> recompute
dense_bytes = 0
+ # Model-owned allocations that are retained once, independently of the
+ # number of per-layer cache slots. Qwen3.8's native FP8 path uses this for
+ # its normalized scale bank; fallback accounting remains conservative.
+ expert_fixed_bytes = 0
expert_groups = {}
tensor_names = set()
for shard in shards:
@@ -122,7 +127,13 @@ def analyze_model(model):
key = (layer, expert)
expert_groups[key] = expert_groups.get(key, 0) + byte_count
else:
- dense_bytes += resident_contribution(resolved, name, size, dtype)
+ fixed_bytes = fixed_resident_contribution(
+ resolved, name, size, dtype)
+ if fixed_bytes:
+ expert_fixed_bytes += fixed_bytes
+ else:
+ dense_bytes += resident_contribution(
+ resolved, name, size, dtype)
layer_sizes = {}
for (layer, _), size in expert_groups.items():
@@ -160,6 +171,7 @@ def analyze_model(model):
"shards": len(shards),
"model_bytes": model_bytes,
"dense_bytes": dense_bytes,
+ "expert_fixed_bytes": expert_fixed_bytes,
"expert_bytes": sum(expert_groups.values()),
"expert_count": len(expert_groups),
"expert_layers": len(per_layer),
@@ -853,7 +865,12 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
max_expert = info["max_expert_bytes"] or typical
kv_bytes = (geometry.context_state_bytes + geometry.fixed_state_bytes) * kv_slots
kv_buffer = geometry.workspace_bytes
- runtime_bytes = int(1.2 * GB + 2.5 * GB + 64 * max_expert + kv_bytes + kv_buffer)
+ # expert_fixed_bytes is retained once per model (currently Qwen3.8's
+ # normalized FP8 scale bank), unlike per_cap_bytes which is paid for
+ # every cache slot. Include it in the resident runtime reservation so the
+ # selected capacity cannot overrun the model's actual allocation.
+ runtime_bytes = int(1.2 * GB + 2.5 * GB + 64 * max_expert +
+ info["expert_fixed_bytes"] + kv_bytes + kv_buffer)
per_cap = info["per_cap_bytes"]
configured_experts = geometry.configured_experts
@@ -973,6 +990,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
"ram": {"role": "resident+warm-experts", "available_bytes": available_memory,
"budget_bytes": ram_budget, "dense_bytes": info["dense_bytes"],
"runtime_bytes": runtime_bytes,
+ "expert_fixed_bytes": info["expert_fixed_bytes"],
"sequence_state_bytes": geometry.context_state_bytes,
"fixed_state_bytes": geometry.fixed_state_bytes,
"workspace_bytes": geometry.workspace_bytes,
diff --git a/c/tests/test_edge_adapters_real.c b/c/tests/test_edge_adapters_real.c
index a8c27038..af777edd 100644
--- a/c/tests/test_edge_adapters_real.c
+++ b/c/tests/test_edge_adapters_real.c
@@ -52,12 +52,14 @@ static int register_all(void) {
coli_kimi_segment_adapter_register() ||
coli_olmoe_segment_adapter_register() ||
coli_qwen36_segment_adapter_register() ||
+ coli_qwen38_segment_adapter_register() ||
coli_deepseek_v4_segment_adapter_register() ||
coli_glm_edge_adapter_register() ||
coli_inkling_edge_adapter_register() ||
coli_kimi_edge_adapter_register() ||
coli_olmoe_edge_adapter_register() ||
coli_qwen36_edge_adapter_register() ||
+ coli_qwen38_edge_adapter_register() ||
coli_deepseek_v4_edge_adapter_register();
}
@@ -115,17 +117,21 @@ int main(int argc, char **argv) {
REQUIRE(edge_cap.flags & COLI_EDGE_CAP_GREEDY,
"real adapter does not expose the model head");
- static const char tokenizer_probe[] = "edge";
+ /* The Qwen3.8 math fixture has a deliberately tiny 64-row vocabulary.
+ * Its fixture tokenizer can still round-trip ASCII punctuation, while the
+ * larger family fixtures retain the more diagnostic multi-byte probe. */
+ const char *tokenizer_probe=!strcmp(family,"qwen38")?"!":"edge";
+ size_t tokenizer_probe_bytes=strlen(tokenizer_probe);
size_t probe_count = 0, probe_bytes = 0;
REQUIRE(coli_edge_tokenize(edge, tokenizer_probe,
- sizeof(tokenizer_probe) - 1u,
+ tokenizer_probe_bytes,
NULL, 0, &probe_count,
error, sizeof(error)) == 0 && probe_count,
"tokenizer sizing pass failed");
probe_ids = malloc(probe_count * sizeof(*probe_ids));
REQUIRE(probe_ids != NULL, "out of memory for tokenizer probe");
REQUIRE(coli_edge_tokenize(edge, tokenizer_probe,
- sizeof(tokenizer_probe) - 1u,
+ tokenizer_probe_bytes,
probe_ids, probe_count, &probe_count,
error, sizeof(error)) == 0,
"tokenizer encode failed");
@@ -139,7 +145,7 @@ int main(int argc, char **argv) {
probe_text, probe_bytes + 1u, &probe_bytes,
error, sizeof(error)) == 0,
"detokenizer decode failed");
- REQUIRE(probe_bytes == sizeof(tokenizer_probe) - 1u &&
+ REQUIRE(probe_bytes == tokenizer_probe_bytes &&
!memcmp(probe_text, tokenizer_probe, probe_bytes),
"tokenizer round-trip differs");
free(probe_text); probe_text = NULL;
diff --git a/c/tests/test_edge_adapters_registration.c b/c/tests/test_edge_adapters_registration.c
index 3568adf9..846a7eac 100644
--- a/c/tests/test_edge_adapters_registration.c
+++ b/c/tests/test_edge_adapters_registration.c
@@ -10,14 +10,16 @@ int main(void) {
assert(coli_kimi_edge_adapter_register() == 0);
assert(coli_olmoe_edge_adapter_register() == 0);
assert(coli_qwen36_edge_adapter_register() == 0);
+ assert(coli_qwen38_edge_adapter_register() == 0);
assert(coli_deepseek_v4_edge_adapter_register() == 0);
- assert(coli_edge_adapter_count() == 6);
+ assert(coli_edge_adapter_count() == 7);
static const char *expected[] = {
- "glm", "inkling", "kimi", "olmoe", "qwen36", "deepseek_v4"
+ "glm", "inkling", "kimi", "olmoe", "qwen36", "qwen38",
+ "deepseek_v4"
};
for (size_t item = 0; item < sizeof(expected) / sizeof(expected[0]); item++)
assert(coli_edge_adapter_lookup(expected[item]) != NULL);
- puts("all six real Edge adapters register: ok");
+ puts("all seven real Edge adapters register: ok");
return 0;
}
diff --git a/c/tests/test_family_registry.py b/c/tests/test_family_registry.py
index c8d0cb7c..69e35aad 100644
--- a/c/tests/test_family_registry.py
+++ b/c/tests/test_family_registry.py
@@ -18,6 +18,7 @@ from family_registry import (
PlannerUnsupportedError,
_build_registry,
expert_contributions,
+ fixed_resident_contribution,
family_for_config,
planner_geometry,
public_metadata,
@@ -192,10 +193,12 @@ class FamilyRegistryTest(unittest.TestCase):
self.assertEqual(resolved.family_config["model_type"], "qwen4_exp_text")
geometry = planner_geometry(resolved, 32)
self.assertEqual(geometry.configured_experts, 512)
+ live_context = 12 * 32 * (2 * 2 * 256 + 128) * 4
self.assertEqual(geometry.context_state_bytes,
- 12 * 32 * (2 * 2 * 256 + 128) * 4)
- expected_fixed = (36 * (48 * 128 * 128 + 10240 * 3) * 4 +
- 10240 * 9 * 4 + 2 * 8)
+ live_context + 32 * 4)
+ live_fixed = (36 * (48 * 128 * 128 + 10240 * 3) * 4 +
+ 10240 * 9 * 4 + 2 * 8)
+ expected_fixed = live_fixed * 2 + 248320 * 4
self.assertEqual(geometry.fixed_state_bytes, expected_fixed)
base_row = 4 * 2560 + 2 * 2560 + 4
gated_residual_row = base_row + 2 * 4 * 2560 + 320
@@ -209,8 +212,20 @@ class FamilyRegistryTest(unittest.TestCase):
ple_fixed = 2560 + 5 * 4 * 2560 + 2560
qsa_fixed = 4 * 128 + 128 + 2048 + 4 - 1 + max(
2 * (32 // 4), 256 + 2048 + 4 - 1)
- expected_workspace = (32 * max(gated_residual_row, qsa_row, ple_row) +
- max(moe_fixed, gdn_fixed, ple_fixed, qsa_fixed)) * 4
+ legacy_workspace = (
+ 32 * max(gated_residual_row, qsa_row, ple_row) +
+ max(moe_fixed, gdn_fixed, ple_fixed, qsa_fixed)) * 4
+ moe_assignment_bytes = 2 * (2560 + 640) * 4 + 8 + 2 * 4
+ moe_row_bytes = (512 + 3 * 640 + 2560 + 1) * 4 + \
+ 10 * moe_assignment_bytes
+ moe_fixed_bytes = 512 * (4 * 4 + 8) + 4
+ moe_chunk_bytes = moe_fixed_bytes + 32 * moe_row_bytes
+ delta_row_bytes = (gdn_width + 2 * 48 * 128 + 2 * 48) * 4
+ delta_fixed_bytes = (gdn_width + 2 * 48 * 128 + 48 * 128) * 4
+ delta_chunk_bytes = delta_fixed_bytes + 32 * delta_row_bytes
+ batched_workspace = 32 * base_row * 4 + max(
+ moe_chunk_bytes, delta_chunk_bytes)
+ expected_workspace = max(legacy_workspace, batched_workspace)
self.assertEqual(geometry.workspace_bytes, expected_workspace)
family = resolved.descriptor
@@ -257,9 +272,12 @@ class FamilyRegistryTest(unittest.TestCase):
self.assertEqual(expert_contributions(
resolved, prefix + ".down_proj.weight", 512, "F16"),
((7, 23, 1024),))
+ # FP8 sidecars are normalized into a fixed per-model scale bank, not
+ # copied into every cache slot.
self.assertEqual(expert_contributions(
- resolved, prefix + ".down_proj.weight_scale_inv", 4, "F32"),
- ((7, 23, 4),))
+ resolved, prefix + ".down_proj.weight_scale_inv", 4, "F32"), ())
+ self.assertEqual(fixed_resident_contribution(
+ resolved, prefix + ".down_proj.weight_scale_inv", 4, "F32"), 4)
with self.assertRaisesRegex(ValueError, "unsupported dtype/size"):
expert_contributions(
resolved, prefix + ".down_proj.weight_scale_inv", 16, "F32")
diff --git a/c/tests/test_qwen38_native_weights.c b/c/tests/test_qwen38_native_weights.c
index 9889ded6..2164dc63 100644
--- a/c/tests/test_qwen38_native_weights.c
+++ b/c/tests/test_qwen38_native_weights.c
@@ -170,8 +170,9 @@ static int write_fp8_fixture(const char *directory,int mixed,int adjacent){
static int init_fixture_model(Model *model,const char *directory,int native_fp8){
memset(model,0,sizeof(*model));model->c.hidden=2;model->c.inter=3;
- model->c.experts=2;model->c.layers=1;model->range_begin=0;model->range_end=1;
- model->native_fp8=native_fp8;model->native_bf16=1;
+ model->c.experts=2;model->c.topk=2;model->c.shared_inter=2;
+ model->c.layers=1;model->range_begin=0;model->range_end=1;
+ model->native_fp8=native_fp8;model->native_bf16=1;model->prefill_batch=1;
snprintf(model->prefix,sizeof model->prefix,"model");
st_init(&model->S,directory);
model->cache=(LCache*)calloc(1,sizeof(*model->cache));
@@ -201,6 +202,39 @@ static void destroy_fixture_model(Model *model){
memset(model,0,sizeof(*model));
}
+static int init_fixture_moe_layer(Model *model){
+ static const float router[4]={1.0f,-1.0f,-1.0f,1.0f};
+ static const float shared_gate[4]={0.25f,-0.5f,0.75f,0.125f};
+ static const float shared_up[4]={-0.25f,0.5f,0.375f,0.625f};
+ static const float shared_down[4]={0.5f,-0.125f,0.25f,0.75f};
+ static const float gate[2]={0.2f,-0.35f};
+ Layer *layer=(Layer*)calloc(1,sizeof(*layer));
+ if(!layer)return -1;
+ q38_weight_reserve(&layer->router,Q38_WEIGHT_F32,2,2);
+ q38_weight_reserve(&layer->sh_g,Q38_WEIGHT_F32,2,2);
+ q38_weight_reserve(&layer->sh_u,Q38_WEIGHT_F32,2,2);
+ q38_weight_reserve(&layer->sh_d,Q38_WEIGHT_F32,2,2);
+ memcpy(layer->router.data,router,sizeof router);
+ memcpy(layer->sh_g.data,shared_gate,sizeof shared_gate);
+ memcpy(layer->sh_u.data,shared_up,sizeof shared_up);
+ memcpy(layer->sh_d.data,shared_down,sizeof shared_down);
+ layer->sh_gate=(float*)malloc(sizeof gate);
+ if(!layer->sh_gate){
+ q38_weight_free(&layer->router);q38_weight_free(&layer->sh_g);
+ q38_weight_free(&layer->sh_u);q38_weight_free(&layer->sh_d);
+ free(layer);return -1;
+ }
+ memcpy(layer->sh_gate,gate,sizeof gate);model->L=layer;
+ return 0;
+}
+
+static void destroy_fixture_moe_layer(Model *model){
+ if(!model->L)return;
+ q38_weight_free(&model->L[0].router);q38_weight_free(&model->L[0].sh_g);
+ q38_weight_free(&model->L[0].sh_u);q38_weight_free(&model->L[0].sh_d);
+ free(model->L[0].sh_gate);free(model->L);model->L=NULL;
+}
+
static uint8_t fixture_fp8_byte(int expert,int projection,int index){
return (uint8_t)(0x20+expert*0x18+projection*7+index);
}
@@ -367,7 +401,158 @@ cleanup:
destroy_fixture_model(&model);return result;
}
+static int check_malformed_scale_metadata(const char *directory){
+ Model model;if(init_fixture_model(&model,directory,1))return -1;
+ char name[320];
+ q38_name(&model,name,sizeof name,0,
+ "mlp.experts.0.gate_proj.weight_scale_inv");
+ st_tensor *scale=st_find(&model.S,name);int result=-1;
+ if(scale){
+ int64_t original_bytes=scale->nbytes;
+ scale->nbytes++;
+ uint64_t bytes=0,fixed=0;unsigned kinds=0;
+ if(q38_segment_expert_layout(&model,0,1,&bytes,&fixed,&kinds)&&
+ !q38_prepare_expert_scale_bank(&model,0))result=0;
+ scale->nbytes=original_bytes;
+ }
+ destroy_fixture_model(&model);return result;
+}
+
+static int check_moe_prefill_parity(const char *directory,int native_fp8,
+ int cache_capacity){
+ static const float input[6]={0.5f,-0.25f,0.5f,-0.25f,0.5f,-0.25f};
+ Model prefill={0},decode={0};float batched[6],serial[6];int result=-1;
+ if(init_fixture_model(&prefill,directory,native_fp8) ||
+ init_fixture_model(&decode,directory,native_fp8) ||
+ init_fixture_moe_layer(&prefill) || init_fixture_moe_layer(&decode))
+ goto cleanup;
+ prefill.c.norm_topk=1;decode.c.norm_topk=1;
+ if(cache_capacity!=1&&q38_segment_cache_resize(&prefill,cache_capacity))
+ goto cleanup;
+ if(cache_capacity!=1&&q38_segment_cache_resize(&decode,cache_capacity))
+ goto cleanup;
+ prefill.expert_parallel_reads=1;
+ q38_moe(&prefill,prefill.L,0,input,3,batched);
+ for(int row=0;row<3;row++)
+ q38_moe_decode(&decode,decode.L,0,input+(int64_t)row*2,1,
+ serial+(int64_t)row*2);
+ if(memcmp(batched,serial,sizeof batched)){
+ fprintf(stderr,"prefill parity mismatch native=%d cap=%d:\n",native_fp8,cache_capacity);
+ for(int i=0;i<6;i++)fprintf(stderr," %d %.9g %.9g\n",i,batched[i],serial[i]);
+ goto cleanup;
+ }
+ /* Every row routes to both experts. The grouped prefill loads each expert
+ * once, even with a one-slot cache; serial decode must reload both experts
+ * for every row. Native FP8 uses two physical payload ranges per load. */
+ uint64_t reads_per_expert=native_fp8?2:3;
+ uint64_t expected_decode_misses=cache_capacity>=2?2:6;
+ uint64_t expected_decode_reads=expected_decode_misses*reads_per_expert;
+ if(prefill.miss!=2||decode.miss!=expected_decode_misses||
+ prefill.expert_weight_reads!=2*reads_per_expert||
+ decode.expert_weight_reads!=expected_decode_reads){
+ fprintf(stderr,"prefill metrics mismatch native=%d cap=%d: miss %llu/%llu reads %llu/%llu\n",
+ native_fp8,cache_capacity,(unsigned long long)prefill.miss,
+ (unsigned long long)decode.miss,(unsigned long long)prefill.expert_weight_reads,
+ (unsigned long long)decode.expert_weight_reads);
+ goto cleanup;
+ }
+ if(cache_capacity>=2&&
+ (prefill.hits!=0||decode.hits!=4||
+ prefill.expert_parallel_batches!=(native_fp8?1u:0u))){
+ fprintf(stderr,"prefill cache metrics mismatch native=%d cap=%d: hits %llu/%llu batches %llu\n",
+ native_fp8,cache_capacity,(unsigned long long)prefill.hits,
+ (unsigned long long)decode.hits,(unsigned long long)prefill.expert_parallel_batches);
+ goto cleanup;
+ }
+ result=0;
+cleanup:
+ destroy_fixture_moe_layer(&prefill);destroy_fixture_moe_layer(&decode);
+ destroy_fixture_model(&prefill);destroy_fixture_model(&decode);return result;
+}
+
+static int check_mixed_moe_prefill_parity(const char *directory){
+ static const float input[6]={0.5f,-0.25f,0.5f,-0.25f,0.5f,-0.25f};
+ Model prefill={0},decode={0};float batched[6],serial[6];int result=-1;
+ if(init_fixture_model(&prefill,directory,1)||
+ init_fixture_model(&decode,directory,1)||
+ init_fixture_moe_layer(&prefill)||init_fixture_moe_layer(&decode))
+ goto cleanup;
+ prefill.c.norm_topk=1;decode.c.norm_topk=1;
+ q38_moe(&prefill,prefill.L,0,input,3,batched);
+ for(int row=0;row<3;row++)
+ q38_moe_decode(&decode,decode.L,0,input+(int64_t)row*2,1,
+ serial+(int64_t)row*2);
+ if(!memcmp(batched,serial,sizeof batched)&&prefill.miss==2&&decode.miss==6)
+ result=0;
+cleanup:
+ destroy_fixture_moe_layer(&prefill);destroy_fixture_moe_layer(&decode);
+ destroy_fixture_model(&prefill);destroy_fixture_model(&decode);return result;
+}
+
+static void fill_test_weight(Q38Weight *weight,int rows,int columns,int salt){
+ q38_weight_reserve(weight,Q38_WEIGHT_F32,rows,columns);
+ float *values=(float*)weight->data;
+ for(int index=0;index<rows*columns;index++)
+ values[index]=(float)(((index+1)*(salt+3))%17-8)/16.f;
+}
+
+/* Projection batching must not alter the causal DeltaNet recurrence. Compare
+ * a three-row chunk with three one-row calls, including both retained states. */
+static int check_deltanet_prefill_parity(void){
+ enum { ROWS=3,H=2,VH=1,KH=1,KD=2,VD=2,V=VH*VD,CD=2*KH*KD+V,CK=2 };
+ Cfg config={0};
+ config.hidden=H;config.dn_vheads=VH;config.dn_kheads=KH;
+ config.dn_kdim=KD;config.dn_vdim=VD;config.dn_conv_dim=CD;
+ config.dn_convk=CK;config.eps=1e-6f;
+ Layer layer={0};
+ fill_test_weight(&layer.dn_qkv,CD,H,1);
+ fill_test_weight(&layer.dn_z,V,H,2);
+ fill_test_weight(&layer.dn_b,VH,H,3);
+ fill_test_weight(&layer.dn_a,VH,H,4);
+ fill_test_weight(&layer.dn_out,H,V,5);
+ layer.dn_conv=(float*)malloc((size_t)CD*CK*sizeof(float));
+ layer.dn_dtbias=(float*)malloc(VH*sizeof(float));
+ layer.dn_alog=(float*)malloc(VH*sizeof(float));
+ layer.dn_norm=(float*)malloc(VD*sizeof(float));
+ if(!layer.dn_conv||!layer.dn_dtbias||!layer.dn_alog||!layer.dn_norm)
+ goto fail;
+ for(int index=0;index<CD*CK;index++)
+ layer.dn_conv[index]=(float)((index%7)-3)/16.f;
+ layer.dn_dtbias[0]=0.125f;layer.dn_alog[0]=-0.75f;
+ layer.dn_norm[0]=0.875f;layer.dn_norm[1]=1.125f;
+
+ float recurrence_a[VH*KD*VD]={0},recurrence_b[VH*KD*VD]={0};
+ float convolution_a[CD*(CK-1)]={0},convolution_b[CD*(CK-1)]={0};
+ float *recurrence_rows_a[1]={recurrence_a},*recurrence_rows_b[1]={recurrence_b};
+ float *convolution_rows_a[1]={convolution_a},*convolution_rows_b[1]={convolution_b};
+ Model batched={0},serial={0};
+ batched.c=config;serial.c=config;
+ batched.prefill_batch=1;
+ batched.DN_rec=recurrence_rows_a;serial.DN_rec=recurrence_rows_b;
+ batched.DN_conv=convolution_rows_a;serial.DN_conv=convolution_rows_b;
+ float input[ROWS*H]={0.5f,-0.25f,-0.75f,0.625f,0.125f,0.875f};
+ float output_a[ROWS*H],output_b[ROWS*H];
+ q38_deltanet(&batched,&layer,0,input,ROWS,output_a);
+ for(int row=0;row<ROWS;row++)
+ q38_deltanet(&serial,&layer,0,input+(int64_t)row*H,1,
+ output_b+(int64_t)row*H);
+ if(memcmp(output_a,output_b,sizeof output_a)||
+ memcmp(recurrence_a,recurrence_b,sizeof recurrence_a)||
+ memcmp(convolution_a,convolution_b,sizeof convolution_a))goto fail;
+
+ q38_weight_free(&layer.dn_qkv);q38_weight_free(&layer.dn_z);
+ q38_weight_free(&layer.dn_b);q38_weight_free(&layer.dn_a);
+ q38_weight_free(&layer.dn_out);free(layer.dn_conv);free(layer.dn_dtbias);
+ free(layer.dn_alog);free(layer.dn_norm);return 0;
+fail:
+ q38_weight_free(&layer.dn_qkv);q38_weight_free(&layer.dn_z);
+ q38_weight_free(&layer.dn_b);q38_weight_free(&layer.dn_a);
+ q38_weight_free(&layer.dn_out);free(layer.dn_conv);free(layer.dn_dtbias);
+ free(layer.dn_alog);free(layer.dn_norm);return -1;
+}
+
static int check_segment_failure_outputs(void){
+ Model partial={0};partial.c.layers=1;q38_model_free(&partial);
char error[128];void *output=(void*)(uintptr_t)1;
ColiSegmentCapabilities capabilities;
ColiSegmentEngineOptions engine_options={
@@ -439,6 +624,7 @@ int main(void){
CHECK(check_mixed_layout(mixed_directory,0,72,
Q38_EXPERT_FP8_EXPANDED|Q38_EXPERT_BF16,
"qwen38/mixed-expert-arithmetic-06/cpu-v1")==0);
+ CHECK(check_mixed_moe_prefill_parity(mixed_directory)==0);
path_length=snprintf(path,sizeof path,"%s/model.safetensors",mixed_directory);
CHECK(path_length>0&&(size_t)path_length<sizeof path);
CHECK(remove(path)==0);CHECK(rmdir(mixed_directory)==0);
@@ -449,6 +635,11 @@ int main(void){
CHECK(check_fixture_mode(adjacent_directory,1,1)==0);
CHECK(check_fixture_mode(adjacent_directory,0,0)==0);
CHECK(check_parallel_batch(adjacent_directory)==0);
+ CHECK(check_malformed_scale_metadata(adjacent_directory)==0);
+ CHECK(check_moe_prefill_parity(adjacent_directory,1,1)==0);
+ CHECK(check_moe_prefill_parity(adjacent_directory,1,2)==0);
+ CHECK(check_moe_prefill_parity(adjacent_directory,0,1)==0);
+ CHECK(check_deltanet_prefill_parity()==0);
path_length=snprintf(path,sizeof path,"%s/model.safetensors",adjacent_directory);
CHECK(path_length>0&&(size_t)path_length<sizeof path);
CHECK(remove(path)==0);CHECK(rmdir(adjacent_directory)==0);
diff --git a/c/tests/test_qwen38_prefix.c b/c/tests/test_qwen38_prefix.c
new file mode 100644
index 00000000..0753956e
--- /dev/null
+++ b/c/tests/test_qwen38_prefix.c
@@ -0,0 +1,89 @@
+/* Qwen3.8 hybrid prompt-prefix state: exact identity, extension, and reset.
+ * The fabricated model exercises the same snapshot/restore helpers used by
+ * SERVE without loading a checkpoint or depending on a tokenizer. */
+#define _GNU_SOURCE
+#define QWEN38_NO_MAIN
+#define QWEN38_TEST_SERVE
+#include "../qwen38.c"
+
+#define CHECK(x) do { if(!(x)){ \
+ fprintf(stderr,"%s:%d: check failed: %s\n",__FILE__,__LINE__,#x);return 1; \
+} } while(0)
+
+static void fill_state(Model *m,float value){
+ for(int i=0;i<m->c.layers;i++)if(!m->c.is_attn[i]){
+ for(size_t j=0;j<g_q38_prefix.rec_cells;j++)m->DN_rec[i][j]=value;
+ for(size_t j=0;j<g_q38_prefix.conv_cells;j++)m->DN_conv[i][j]=value+1.f;
+ }
+ for(size_t j=0;j<g_q38_prefix.ple_cells;j++)m->PLE_conv_state[j]=value+2.f;
+ m->ple_history_len=2;m->ple_history[0]=41;m->ple_history[1]=42;
+}
+
+static int check_state(const Model *m,float value){
+ for(int i=0;i<m->c.layers;i++)if(!m->c.is_attn[i]){
+ for(size_t j=0;j<g_q38_prefix.rec_cells;j++)if(m->DN_rec[i][j]!=value)return 0;
+ for(size_t j=0;j<g_q38_prefix.conv_cells;j++)if(m->DN_conv[i][j]!=value+1.f)return 0;
+ }
+ for(size_t j=0;j<g_q38_prefix.ple_cells;j++)if(m->PLE_conv_state[j]!=value+2.f)return 0;
+ return m->ple_history_len==2&&m->ple_history[0]==41&&m->ple_history[1]==42;
+}
+
+static void free_fake(Model *m){
+ q38_prefix_cache_release(m);
+ for(int i=0;i<m->c.layers;i++){
+ free(m->DN_rec[i]);free(m->DN_conv[i]);
+ free(m->K[i]);free(m->V[i]);free(m->IK[i]);
+ }
+ free(m->DN_rec);free(m->DN_conv);free(m->K);free(m->V);free(m->IK);
+ free(m->PLE_conv_state);free(m->ple_history);free(m->c.is_attn);
+}
+
+int main(void){
+ Model m={0};
+ m.c.layers=2;m.c.vocab=8;m.c.dn_vheads=1;m.c.dn_kdim=2;m.c.dn_vdim=2;
+ m.c.dn_conv_dim=2;m.c.dn_convk=3;m.c.ple_layer=1;m.c.hc_width=2;
+ m.c.ple_convk=3;m.c.ngram_size=1;m.c.is_attn=calloc(2,1);CHECK(m.c.is_attn);
+ m.c.is_attn[1]=1;
+ m.DN_rec=calloc(2,sizeof(float*));m.DN_conv=calloc(2,sizeof(float*));
+ m.K=calloc(2,sizeof(float*));m.V=calloc(2,sizeof(float*));m.IK=calloc(2,sizeof(float*));
+ CHECK(m.DN_rec&&m.DN_conv&&m.K&&m.V&&m.IK);
+ m.DN_rec[0]=calloc(4,sizeof(float));m.DN_conv[0]=calloc(4,sizeof(float));
+ m.PLE_conv_state=calloc(4,sizeof(float));m.ple_history=calloc(2,sizeof(int64_t));
+ CHECK(m.DN_rec[0]&&m.DN_conv[0]&&m.PLE_conv_state&&m.ple_history);
+ CHECK(q38_prefix_cache_layout(&m));
+
+ /* QSA rows are position-indexed. Verify growth keeps the old prefix. */
+ m.c.kv_heads=1;m.c.head_dim=2;m.c.idx_dim=2;m.kv_cap=3;m.kv_len=3;
+ m.K[1]=malloc(6*sizeof(float));m.V[1]=malloc(6*sizeof(float));m.IK[1]=malloc(6*sizeof(float));
+ CHECK(m.K[1]&&m.V[1]&&m.IK[1]);
+ for(int i=0;i<6;i++){m.K[1][i]=(float)i;m.V[1][i]=(float)(10+i);m.IK[1][i]=(float)(20+i);}
+
+ const int first[]={1,2,3};float first_logits[8];
+ for(int i=0;i<8;i++)first_logits[i]=(float)(100+i);
+ fill_state(&m,7.f);CHECK(q38_prefix_cache_save(&m,first,3,first_logits));
+ fill_state(&m,99.f);m.kv_len=3;
+ const int extension[]={1,2,3,4};
+ CHECK(q38_prefix_restore(&m,extension,4)==3);CHECK(check_state(&m,7.f));
+ CHECK(q38_serve_ensure_kv(&m,6));CHECK(m.kv_cap==6&&m.kv_len==3);
+ for(int i=0;i<6;i++){CHECK(m.K[1][i]==(float)i);CHECK(m.V[1][i]==(float)(10+i));CHECK(m.IK[1][i]==(float)(20+i));}
+
+ /* An exact prompt restores the recurrent state and its saved logits. */
+ fill_state(&m,55.f);CHECK(q38_prefix_restore(&m,first,3)==3);CHECK(check_state(&m,7.f));
+ fill_state(&m,56.f);CHECK(q38_prefix_restore(&m,extension,4)==3);CHECK(check_state(&m,7.f));
+ CHECK(q38_prefix_cache_save(&m,extension,4,first_logits));
+ fill_state(&m,88.f);CHECK(q38_prefix_restore(&m,extension,4)==4);CHECK(check_state(&m,7.f));
+ const float *cached=q38_prefix_cached_logits(&m);CHECK(cached&&cached[0]==100.f&&cached[7]==107.f);
+
+ /* A mismatch is never restored; the serve miss path invalidates then
+ * resets the live recurrent state before feeding the new prompt. */
+ const int shorter[]={1,2,3};CHECK(q38_prefix_restore(&m,shorter,3)==0);
+ const int mismatch[]={1,2,4};CHECK(q38_prefix_restore(&m,mismatch,3)==0);
+ q38_prefix_cache_invalidate();reset_recurrent(&m);m.kv_len=0;
+ CHECK(!g_q38_prefix.valid&&m.kv_len==0&&m.ple_history_len==0);
+ for(size_t i=0;i<g_q38_prefix.rec_cells;i++)CHECK(m.DN_rec[0][i]==0.f);
+ for(size_t i=0;i<g_q38_prefix.conv_cells;i++)CHECK(m.DN_conv[0][i]==0.f);
+ for(size_t i=0;i<g_q38_prefix.ple_cells;i++)CHECK(m.PLE_conv_state[i]==0.f);
+ free_fake(&m);
+ puts("qwen38 prefix: exact reuse, extension, mismatch reset, state restore: ok");
+ return 0;
+}
diff --git a/c/tests/test_qwen38_serve_framing.c b/c/tests/test_qwen38_serve_framing.c
index 9c3ec77e..3f77ffb2 100644
--- a/c/tests/test_qwen38_serve_framing.c
+++ b/c/tests/test_qwen38_serve_framing.c
@@ -28,9 +28,16 @@ int main(void){
FILE *in=input_with("SUBMIT req 0 5 2 0 1\nHello\n");CHECK(in!=NULL);
ServeReq q={0};
CHECK(serve_read_req(in,out,&q,NULL)==2);
- CHECK(!strcmp(q.id,"req")&&q.max_tok==2&&q.plen==5&&!strcmp(q.payload,"Hello"));
+ CHECK(!strcmp(q.id,"req")&&q.slot==0&&q.max_tok==2&&q.plen==5&&!strcmp(q.payload,"Hello"));
free(q.payload);fclose(in);
+ /* The wire codec must consume a valid frame before the one-slot policy
+ * rejects a nonzero cache slot, so the following command remains aligned. */
+ in=input_with("SUBMIT wrong 7 5 1 0 1\nHello\nCANCEL wrong\n");CHECK(in!=NULL);
+ CHECK(serve_read_req(in,out,NULL,NULL)==0);
+ CHECK(serve_read_req(in,out,NULL,"wrong")==3);
+ fclose(in);
+
in=input_with("STOP req\nCANCEL req\n");CHECK(in!=NULL);
CHECK(serve_read_req(in,out,NULL,"req")==1);
CHECK(serve_read_req(in,out,NULL,"req")==3);
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index bed759cb..c5fb13f5 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -677,7 +677,10 @@ memInfo.free: 23.50 GB (97%)
write_shard(self.model / "model.safetensors", tensors)
analysis = analyze_model(self.model)
self.assertEqual(analysis["dense_bytes"], 256)
- self.assertEqual(analysis["expert_bytes"], 492)
+ # The three native FP8 sidecars are retained once in the normalized
+ # scale bank, not once per cache slot.
+ self.assertEqual(analysis["expert_fixed_bytes"], 12)
+ self.assertEqual(analysis["expert_bytes"], 480)
# A slot can receive either expert. It must use the larger retained
# representation, not the median of unlike source dtypes.
self.assertEqual(analysis["expert_bytes_by_layer"], {0: 384})
diff --git a/c/tools/make_edge_tiny_tokenizer.py b/c/tools/make_edge_tiny_tokenizer.py
index e7e714b4..55776c1c 100644
--- a/c/tools/make_edge_tiny_tokenizer.py
+++ b/c/tools/make_edge_tiny_tokenizer.py
@@ -34,8 +34,8 @@ def main() -> int:
parser.add_argument("model_dir", type=Path)
parser.add_argument("--vocab-size", type=int, required=True)
args = parser.parse_args()
- if args.vocab_size < 128:
- raise SystemExit("Edge fixture vocabulary must contain at least 128 IDs")
+ if args.vocab_size < 34:
+ raise SystemExit("Edge fixture vocabulary must contain at least 34 IDs")
args.model_dir.mkdir(parents=True, exist_ok=True)
symbols = byte_symbols()
vocab = {symbols[token]: token for token in range(min(256, args.vocab_size))}
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 05464a1c..f82d7391 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -16,7 +16,7 @@ what follows, but the sister engines read their own:
| `kimi_k3` | `c/kimi_k3.c` | the `K3_*` family β€” see [Kimi K3 engine](#kimi-k3-engine-kimi_k3) |
| `inkling` | `c/inkling.c` | `INK_*`, plus `CTX_MAX`, `PIN_N`, `REP_PEN`, `GPU_DEV`, `NOGPU` β€” see [Inkling engine](#inkling-engine-inkling) |
| `qwen36` | `c/qwen36.c` | `QWEN_*`, `Q36_*`, and its dense/CUDA-tier controls β€” see [Qwen3.6 engine](#qwen36-engine-qwen36) |
-| `qwen38` | `c/qwen38.c` | `Q38_MAXT`, `Q38_EOS`, `Q38_NATIVE_FP8`, `Q38_NATIVE_BF16`, `COLI_TIMERS` β€” see [Qwen3.8 engine](#qwen38-engine-qwen38) |
+| `qwen38` | `c/qwen38.c` | `Q38_MAXT`, `Q38_EOS`, `Q38_NATIVE_FP8`, `Q38_NATIVE_BF16`, `Q38_PREFILL_BATCH`, `COLI_TIMERS` β€” see [Qwen3.8 engine](#qwen38-engine-qwen38) |
| `olmoe` | `c/olmoe.c` | `HOT`, `WIDE`, `SMOOTH`, `CONF_LIMIT`, `MAX_NEW`, `CHAT`, `EXPERT_DROP`, `WARMUP` β€” see [OLMoE engine](#olmoe-engine-olmoe) |
| `deepseek_v4` | `c/deepseek_v4.c` | `CTX`, the `V4_*` / `DSV4_*` families and the two `COLI_CUDA_*_BATCH` gates β€” see [DeepSeek V4 engine](#deepseek-v4-engine-deepseek_v4); note that the CUDA section below describes `colibri.c` knobs (`COLI_CUDA`, `CUDA_DENSE`, ...) which the V4 engine does not read β€” its GPU switch is `DSV4_CUDA` |
@@ -383,6 +383,7 @@ checkpoint layout and the text-only capability boundary.
| `Q38_EOS` | tokenizer/config stop IDs | Override the served end-of-sequence token ID for controlled experiments. Normally the engine stops on the tokenizer's `<|im_end|>` / `<|endoftext|>` IDs, falling back to `eos_token_id`. |
| `Q38_NATIVE_FP8` | `1` (on) | Keep routed E4M3 expert bytes and their F32 128Γ—128 block scales native in the LRU. `=0` restores expanded-FP32 slots for A/B validation. |
| `Q38_NATIVE_BF16` | `1` (on) | Keep resident and routed BF16 matrices in two-byte storage while retaining FP32 activations/accumulation. `=0` restores the expanded-FP32 reference. |
+| `Q38_PREFILL_BATCH` | `1` (on) | Route prompt rows in bounded expert-major chunks and batch resident shared-expert/DeltaNet projections. `=0` restores row-at-a-time prompt execution for A/B diagnosis; decode is unchanged. |
| `COLI_TIMERS` | `0` (off) | Set to `1` for the detailed Qwen3.8 phase breakdown on stderr. The shared per-request `PROF` frame is emitted regardless. |
## DeepSeek V4 engine (`deepseek_v4`)
diff --git a/docs/edge-runtime.md b/docs/edge-runtime.md
index d809503a..3a3c8e58 100644
--- a/docs/edge-runtime.md
+++ b/docs/edge-runtime.md
@@ -19,7 +19,7 @@ adapters and keep their existing initialization and inference paths.
External consumers can build the complete CPU-baseline runtime with
`make -C c segment-edge-library`. The resulting
`c/build/segment/libcolibri_segment_edge.a` contains both public runtimes and
-all six built-in adapters. Consumers link it after their own objects and call
+all seven built-in adapters. Consumers link it after their own objects and call
the explicit registration functions below; ordinary Colibri executables do
not link this archive.
@@ -59,6 +59,7 @@ Colibri model family:
| Kimi K3 | hidden plus all AttnRes blocks | tokenizer, final transforms and head; embedding rows stream on demand |
| OLMoE | hidden state | tokenizer, embedding, final norm and head |
| Qwen3.6 | hidden state | tokenizer, embedding, final norm and head |
+| Qwen3.8-Flash-Next | four-stream hyper state | tokenizer, native BF16 embedding, final hyper mixer and native BF16 head |
| DeepSeek V4 | expanded mHC state | tokenizer and small final mHC tensors; BF16 embedding rows and head tiles stream on demand |
Every adapter currently advertises CPU only. Accelerator flags will be added
@@ -77,13 +78,16 @@ returning one greedy token and an optional score per row.
The runtime validates structure sizes, activation geometry, batch limits,
output capacities and cancellation before entering an adapter. Model adapters
-also validate token IDs and checkpoint boundary tensors. Errors are returned
-to the caller; the network layer decides whether to retry, migrate or fall back
-locally.
+also validate token IDs and checkpoint boundary tensors. Operational and
+compatibility errors are returned to the caller; the network layer decides
+whether to retry, migrate or fall back locally. Checkpoint discovery still uses
+the engines' shared fail-closed safetensors/config loaders: a malformed or
+hostile model directory is a process-fatal deployment error, as it is for the
+standalone engines, rather than a recoverable request error.
## All-family release gate
-`make -C c edge-adapters` verifies that all six real adapters coexist in one
+`make -C c edge-adapters` verifies that all seven real adapters coexist in one
runtime. `edge-adapters-real` is the stronger gate: for every family it loads
the real Edge adapter and a full real Segment range, round-trips text, embeds
an independent oracle prompt, performs prefill plus decode and compares three
@@ -114,6 +118,9 @@ python3 tools/make_edge_tiny_tokenizer.py qwen36_edge_src --vocab-size 320
python3 tools/convert_qwen36.py --model qwen36_edge_src \
--out qwen36_edge_i8 --ebits 8 --no-readme
+python3 tools/make_qwen38_tiny.py --out qwen38_tiny
+python3 tools/make_edge_tiny_tokenizer.py qwen38_tiny --vocab-size 64
+
python3 tools/make_deepseek_v4_tiny.py --output deepseek_v4_edge_tiny --force
python3 tools/make_edge_tiny_tokenizer.py deepseek_v4_edge_tiny --vocab-size 128
```
@@ -131,6 +138,8 @@ make -C c edge-adapters-real \
OLMOE_EDGE_REF=olmoe_tiny_src/ref_olmoe.json \
QWEN_EDGE_MODEL=qwen36_edge_i8 \
QWEN_EDGE_REF=qwen36_edge_src/ref_qwen36.json \
+ QWEN38_EDGE_MODEL=qwen38_tiny \
+ QWEN38_EDGE_REF=qwen38_tiny/ref.json \
DEEPSEEK_EDGE_MODEL=deepseek_v4_edge_tiny \
DEEPSEEK_EDGE_REF=deepseek_v4_edge_tiny/ref.json
```
diff --git a/docs/qwen38.md b/docs/qwen38.md
index 77ecf40b..54414039 100644
--- a/docs/qwen38.md
+++ b/docs/qwen38.md
@@ -52,6 +52,21 @@ capacity is the first engine positional argument. The canonical FP8 matmul
decodes values during accumulation rather than materializing three FP32 matrices
per slot. The launcher defaults to one expert slot per layer.
+Prompt execution is expert-major in bounded chunks: it routes up to 32 rows,
+groups their assignments by expert, and consumes cache-sized parallel load
+groups when the complete demand set is larger than the configured LRU. Shared
+expert and DeltaNet projections are batched over the same bounded window;
+DeltaNet convolution and recurrent updates remain token-causal. The private
+chunk workspace is capped at 64 MiB independently of prompt length.
+
+The serve path owns one hybrid prefix slot. When the next prompt begins with
+the exact cached token sequence, QSA K/V/index rows are reused in place and the
+saved DeltaNet/PLE recurrent state is restored before evaluating only the
+extension. An identical prompt also reuses its saved final logits. Mismatched
+or shorter prompts reset every hybrid component rather than guessing at cache
+identity. Decode and cancellation mutate only live state, not the published
+prompt snapshot.
+
QSA caches the two K/V heads and the indexer's raw key. Complete four-token
blocks are pooled and scored, the best 512 blocks are retained, and a causal
tail of up to three tokens is appended. The main 24-head attention then operates
@@ -105,5 +120,8 @@ numerical/performance A/Bs; `Q38_NATIVE_BF16=0` does the same for resident and
routed BF16 matrices. Other model types, unsupported scale encodings, and
incompatible tensor shapes fail during load.
+`Q38_PREFILL_BATCH=0` restores row-at-a-time prompt execution for controlled
+A/B diagnosis. It does not change the single-token decode path.
+
The weights remain covered by the Qwen Community License 1.0 in the downloaded
checkpoint. They are not redistributed by Colibri.
diff --git a/docs/segment-runtime.md b/docs/segment-runtime.md
index e9b51707..80b4eeb5 100644
--- a/docs/segment-runtime.md
+++ b/docs/segment-runtime.md
@@ -85,6 +85,10 @@ sizes and context bounds before calling the adapter.
Positions and model-specific ordering rules remain adapter-owned. A failed or
cancelled run must not be reported as committed by the distributed caller.
Network-level idempotency and duplicate request handling belong above this ABI.
+Cancellation is cooperative, not asynchronous: an adapter may check only at a
+model-safe boundary, and Qwen3.8 currently checks before entering a complete
+Segment run. Callers that need prompt-time interruption should terminate or
+migrate the worker and retry from the last published snapshot.
## Snapshot contract