Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge upstream/dev and resolve PR #165 conflicts

DrewZ27637f0b579c
diff --cc .github/workflows/check.yml
index d1f0e8b6,59968985..10db7be5
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@@ -48,9 -34,17 +48,17 @@@ jobs
make
mingw-w64-ucrt-x86_64-gcc
mingw-w64-ucrt-x86_64-python
- - name: make check
+ - name: make check (includes tiny V4 oracle)
run: make -C c check
+ # The Windows engine as this branch builds it. Small, and it makes an
+ # antivirus false-positive report (#527) verifiable on a PR instead of
+ # only after a release is tagged and published.
+ - uses: actions/upload-artifact@v4
+ with:
+ name: colibri-windows-exe
+ path: c/colibri.exe
+
macos:
# clang; libomp for the threaded path (Makefile falls back to
# single-threaded automatically if it's ever missing).
diff --cc README.md
index c58b72fc,40facb07..07f2c106
--- a/README.md
+++ b/README.md
@@@ -280,30 -441,14 +441,31 @@@ Two things that differ per model, both
| Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) |
| Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
+## DeepSeek V4
+
+The experimental CPU path for **DeepSeek V4 Flash + DSpark** uses native FP4
+experts, automatic RAM planning, and lossless speculative verification. It is
+supported on x86-64 Linux and Windows/MSYS2; other platforms keep validating
+the main colibri engine through `make check`.
+
+```bash
+cd c
+make deepseek-v4
+python ./v4 run --model /path/to/DeepSeek-V4-Flash-DSpark --ram 32 \
+ --stop-sentence "What is the capital of France?"
+```
+
+See [docs/deepseek-v4.md](docs/deepseek-v4.md) for status, benchmarks,
+checkpoint validation, and the committed tiny independent oracle.
+
## What's next
- - **Algorithmic research is active.** The current hierarchy is LRU + a learned
- pin set; the next step is under way — smarter placement and scheduling,
- overlap of CPU and GPU expert execution, and routing-aware speculation.
- Everything lands the way this project always works: measured, reviewed, and
- merged in the open.
+ - **Inference-systems research is the product.** The current hierarchy is LRU +
+ a learned pin set; active work spans model formats, compression, placement,
+ scheduling, I/O, CPU/GPU kernels, heterogeneous overlap, KV state, and
+ routing-aware speculation. The objective is lower hardware requirements and
+ lower cost per useful token. Everything lands the way this project works:
+ measured end to end, reviewed, and developed in the open.
- **More open models.** The tiering algorithm is model-agnostic: any MoE with
routed experts can be staged the same way. GLM-5.2 and OLMoE run today;
support for more open-weight families — **Kimi K2** (Moonshot AI),
diff --cc README.zh-CN.md
index aaf12340,44ca02be..15b5aa23
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@@ -217,23 -283,15 +283,32 @@@ COLI_MODEL=/nvme/glm52_i4 ./coli docto
| 语法强制草稿(结构化输出) | [docs/grammar-draft.md](docs/grammar-draft.md) |
| 环境变量完整清单 | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) |
+## DeepSeek V4
+
+实验性的 **DeepSeek V4 Flash + DSpark** CPU 路径使用原生 FP4 专家、
+自动 RAM 规划和无损推测验证。目前支持 x86-64 Linux 与 Windows/MSYS2;
+其他平台仍通过 `make check` 验证 colibri 主引擎。
+
+```bash
+cd c
+make deepseek-v4
+python ./v4 run --model /path/to/DeepSeek-V4-Flash-DSpark --ram 32 \
+ --stop-sentence "What is the capital of France?"
+```
+
+状态、benchmark、checkpoint 验证和已提交的 tiny 独立 oracle 说明,
+请参阅[中文版 DeepSeek V4 文档](docs/deepseek-v4.zh-CN.md);
+英文原文见 [docs/deepseek-v4.md](docs/deepseek-v4.md)。
+
+ ## 下一步
+
+ - **推理系统研究就是产品。**当前层级采用 LRU 与学习型固定集;正在研究模型格式、压缩、
+ 放置、调度、I/O、CPU/GPU 内核、异构重叠、KV 状态与路由感知推测。目标是降低硬件要求
+ 和每个有效 token 的成本,所有成果都以端到端测量为准、经审查并公开开发。
+ - **支持更多开放模型。**层级算法与模型无关,任何带路由专家的 MoE 都能用相同方式分层。
+ GLM-5.2 与 OLMoE 目前可用;**Kimi K2**、**Qwen3 MoE**、**MiniMax** 等开放权重模型
+ 已列入路线图。
+
## 支持项目
colibrì 最初由一人使用 12 核心、25 GB RAM 的笔记本开发;
diff --cc c/Makefile
index 79770dca,11615113..13ee371f
--- a/c/Makefile
+++ b/c/Makefile
@@@ -199,28 -216,20 +216,39 @@@ els
PYTHON ?= python3
endif
CUDA_OBJ =
+
+# DeepSeek V4 is a separate x86-64/aarch64 runtime. Portable infrastructure
+# tests run everywhere; the engine and token-exact fixture run on x86-64
+# Linux/Windows and aarch64 Linux. SIMD fast paths are gated per-feature
+# inside the sources, so a plain armv8-a build stays correct.
+COLI_V4_SUPPORTED :=
+ifneq (,$(X86_64))
+ifneq (,$(IS_WIN)$(LINUX))
+COLI_V4_SUPPORTED := 1
+endif
+endif
+ifneq (,$(AARCH64))
+ifneq (,$(LINUX))
+COLI_V4_SUPPORTED := 1
+endif
+endif
+
- V4_INFRA_TEST_BINS = tests/test_expert_store_ops$(EXE) \
- tests/test_safetensors_index$(EXE) tests/test_native_quant$(EXE) \
- tests/test_tensor_io$(EXE)
- TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_st_mirror$(EXE) tests/test_st_pread$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_topp$(EXE) tests/test_sample_nan$(EXE) tests/test_temp_env$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_int3$(EXE) tests/test_int3_load$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE) tests/test_dsa_select$(EXE) tests/test_logit_nan$(EXE) tests/test_pipe_block$(EXE) tests/test_e8_kernel$(EXE)
- TEST_BINS += $(V4_INFRA_TEST_BINS)
+ # Gates are exactly the tests that have a build rule further down -- derived from those
+ # rules, so adding a gate means adding your tests/test_*.c and its rule, and nothing else.
+ # There is no shared list to conflict on.
+ #
+ # This replaced a single hand-written TEST_BINS line. c/Makefile appears in most open PRs,
+ # and any two that each added a test conflicted on that line by construction; #386 hit it
+ # twice while being rebased. A first attempt (#731) globbed tests/test_*.c, which was wrong
+ # in the other direction: it promoted files that deliberately have no rule (a branch's
+ # work-in-progress test, test_fse, test_tok, test_tok_kimi, test_vk_mxfp4) into gates, and
+ # they fail to link. Having a rule is the honest definition of "this is a gate".
+ TEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(firstword $(MAKEFILE_LIST)))
-# test_uring has a rule but is Linux-only; it is appended conditionally just below.
-TEST_EXCLUDE = test_uring
++# test_uring is Linux-only. V4 engine tests are appended below only on supported
++# x86-64 Linux/Windows and aarch64 Linux hosts; the V4 infrastructure tests have
++# unconditional rules and therefore remain portable gates.
++TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_config_dspark_compat
+ TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
@@@ -280,43 -313,27 +336,62 @@@ LDFLAGS += -framework Metal -framewor
METAL_OBJ = backend_metal.o
endif
+ # VK=1 adds an opt-in Vulkan compute backend (any Vulkan 1.2 driver, e.g. Mesa/RADV)
+ # for resident expert tensors. Independent of CUDA/HIP: works on GPUs ROCm doesn't
+ # support (Polaris/RX 580); the VRAM-resident int4 expert-group PRIMITIVE measured ~35%
+ # faster than the ROCm/HIP one on an RX 9070 (per-call incl. readback — end-to-end tok/s
+ # depends on what else bounds the box; see docs/vulkan.md, benchmarking notes included).
+ # Needs libvulkan + glslc (shaderc). Default build unchanged (all colibri.c hooks under
+ # #ifdef COLI_VULKAN). The .comp shaders compile to .spv, loaded at runtime from
+ # COLI_VK_SHADERS (default: alongside the binary / ./shaders).
+ VK ?= 0
+ VK_OBJ =
+ VK_SPV =
+ GLSLC ?= glslc
+ ifeq ($(VK),1)
+ CFLAGS += -DCOLI_VULKAN
+ LDFLAGS += -lvulkan
+ VK_OBJ = backend_vulkan.o
+ VK_SPV = shaders/qmatmul.spv shaders/qmatmul_gate_up.spv shaders/attention_absorb.spv shaders/rmsnorm.spv
+ endif
+
all: colibri$(EXE)
+.PHONY: deepseek-v4 deepseek-v4-oracle deepseek-v4-tiny-check deepseek-v4-clean
+ifeq ($(COLI_V4_SUPPORTED),1)
+deepseek-v4:
+ $(MAKE) -f Makefile.deepseek-v4 ARCH=$(ARCH) deepseek-v4
+
+deepseek-v4-clean:
+ $(MAKE) -f Makefile.deepseek-v4 deepseek-v4-clean
+
+deepseek-v4-tiny-check:
+ $(MAKE) -f Makefile.deepseek-v4 deepseek-v4-clean
+ $(MAKE) -f Makefile.deepseek-v4 ARCH=$(if $(strip $(ARCH)),$(ARCH),$(PORTABLE_ARCH)) deepseek-v4
+ $(PYTHON) tests/test_deepseek_v4_tiny.py \
+ --binary ./$(if $(IS_WIN),deepseek_v4.exe,deepseek_v4) \
+ --fixture ./deepseek_v4_tiny
+
+deepseek-v4-oracle: deepseek-v4
+ @test -n "$(MODEL)" || { echo "usage: make deepseek-v4-oracle MODEL=/path/to/checkpoint" >&2; exit 2; }
+ $(PYTHON) tools/make_deepseek_v4_oracle.py --model "$(MODEL)" \
+ --binary ./$(if $(IS_WIN),deepseek_v4.exe,deepseek_v4) \
+ --output tests/deepseek_v4_oracle.json \
+ --memory-gb $(or $(MEMORY_GB),32) \
+ --prompt "$(or $(ORACLE_PROMPT),The capital of France is)" \
+ --validate \
+ --teacher-forcing $(or $(ORACLE_TEACHER_FORCING),32) \
+ --greedy $(or $(ORACLE_GREEDY),20) \
+ --check-dspark
+else
+deepseek-v4 deepseek-v4-oracle:
+ @echo "$@ is supported only on x86-64/aarch64 Linux and Windows/MSYS2" >&2; exit 1
+deepseek-v4-clean:
+ @:
+deepseek-v4-tiny-check:
+ @echo "SKIP deepseek-v4-tiny-check: V4 runtime requires x86-64/aarch64 Linux or Windows/MSYS2"
+endif
+
# phony targets — 'glm' kept for backward compatibility
colibri: colibri$(EXE)
glm: colibri$(EXE)
@@@ -487,72 -593,19 +651,84 @@@ tests/test_i4_acc512$(EXE): tests/test_
tests/test_compat_direct$(EXE): tests/test_compat_direct.c compat.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_expert_store_ops$(EXE): tests/test_expert_store_ops.c expert_store.h tensor.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+tests/test_safetensors_index$(EXE): tests/test_safetensors_index.c safetensors_index.c safetensors_index.h json.h
+ $(CC) $(CFLAGS) safetensors_index.c $< -o $@ $(LDFLAGS)
+
+tests/test_native_quant$(EXE): tests/test_native_quant.c native_quant.c native_quant.h tensor.h
+ $(CC) $(CFLAGS) native_quant.c $< -o $@ $(LDFLAGS)
+
+tests/test_tensor_io$(EXE): tests/test_tensor_io.c tensor_io.c tensor_io.h native_quant.c native_quant.h safetensors_index.c safetensors_index.h tensor.h json.h
+ $(CC) $(CFLAGS) safetensors_index.c native_quant.c tensor_io.c $< \
+ -o $@ $(LDFLAGS)
+
+ifeq ($(COLI_V4_SUPPORTED),1)
+include Makefile.deepseek-v4.units
+V4_TEST_LINK_OBJS = \
+ COLI_V4_UNIT_MATH.o \
+ COLI_V4_UNIT_SPARSE_ATTENTION.o \
+ COLI_V4_UNIT_RESOURCE_PLAN.o \
+ COLI_V4_UNIT_PROMPT.o \
+ $(addsuffix .o,$(V4_TEST_UNITS)) \
+ safetensors_index.o native_quant.o
+
+tests/test_deepseek_v4$(EXE): tests/test_deepseek_v4.c deepseek_v4.c deepseek_v4.h \
+ deepseek_v4_dspark.h Makefile.deepseek-v4.units
+ $(MAKE) -f Makefile.deepseek-v4 ARCH=$(ARCH) deepseek-v4-test-objs \
+ COLI_V4_UNIT_MATH.o COLI_V4_UNIT_SPARSE_ATTENTION.o \
+ COLI_V4_UNIT_RESOURCE_PLAN.o COLI_V4_UNIT_PROMPT.o \
+ safetensors_index.o
+ $(CC) $(CFLAGS) native_quant.c -c -o native_quant.o
+ $(CC) $(CFLAGS) $< $(V4_TEST_LINK_OBJS) -o $@ -pthread $(LDFLAGS)
+
+V4_OWN_DIR = build/ownership
+V4_OWN_CFLAGS = $(CFLAGS) -DCOLI_V4_TEST_HOOKS
+V4_OWNERSHIP_OBJS = \
+ $(V4_OWN_DIR)/COLI_V4_UNIT_RUNTIME.o \
+ $(V4_OWN_DIR)/COLI_V4_UNIT_CONFIG.o \
+ $(V4_OWN_DIR)/safetensors_index.o
+
+$(V4_OWN_DIR):
+ mkdir -p $(V4_OWN_DIR)
+
+$(V4_OWN_DIR)/COLI_V4_UNIT_RUNTIME.o: deepseek_v4.c deepseek_v4.h deepseek_v4_internal.h | $(V4_OWN_DIR)
+ $(CC) $(V4_OWN_CFLAGS) -DCOLI_V4_UNIT_RUNTIME -c deepseek_v4.c -o $@
+
+$(V4_OWN_DIR)/COLI_V4_UNIT_CONFIG.o: deepseek_v4.c deepseek_v4.h deepseek_v4_internal.h | $(V4_OWN_DIR)
+ $(CC) $(V4_OWN_CFLAGS) -DCOLI_V4_UNIT_CONFIG -c deepseek_v4.c -o $@
+
+$(V4_OWN_DIR)/safetensors_index.o: safetensors_index.c safetensors_index.h | $(V4_OWN_DIR)
+ $(CC) $(CFLAGS) -c safetensors_index.c -o $@
+
+tests/test_v4_ownership$(EXE): tests/test_v4_ownership.c $(V4_OWNERSHIP_OBJS)
+ $(CC) $(V4_OWN_CFLAGS) $< $(V4_OWNERSHIP_OBJS) -o $@ -pthread $(LDFLAGS)
+
+COLI_V4_UNIT_CONFIG_DSPARK_COMPAT.o: deepseek_v4_dspark.c deepseek_v4.h \
+ deepseek_v4_dspark.h deepseek_v4_internal.h Makefile.deepseek-v4
+ $(MAKE) -f Makefile.deepseek-v4 ARCH=$(ARCH) $@
+
+tests/test_v4_config_dspark_compat$(EXE): \
+ tests/test_v4_config_dspark_compat.c \
+ COLI_V4_UNIT_CONFIG_DSPARK_COMPAT.o
+ $(CC) $(CFLAGS) $< COLI_V4_UNIT_CONFIG_DSPARK_COMPAT.o \
+ -o $@ $(LDFLAGS)
+endif
+
- tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+ tests/test_route_trace$(EXE): tests/test_route_trace.c route_trace.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+ tests/test_dsa_select$(EXE): tests/test_dsa_select.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+ tests/test_corpus_draft$(EXE): tests/test_corpus_draft.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+ tests/test_cap_precedence$(EXE): tests/test_cap_precedence.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+ tests/test_ssd_probe$(EXE): tests/test_ssd_probe.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_dsa_select is a microbenchmark (old qsort vs new quickselect partial-select, #356),
@@@ -633,5 -702,4 +826,6 @@@ clean
bench: iobench$(EXE)
@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi
-.PHONY: all colibri glm iq3 rans fuzz-rans cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
+.PHONY: all colibri glm deepseek-v4 deepseek-v4-clean deepseek-v4-oracle deepseek-v4-tiny-check \
- cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
++ iq3 rans fuzz-rans cuda-test hip-test gpu-compile cuda-bench cuda-dll portable \
++ test-c test-python test check clean install uninstall bench
diff --cc c/compat.h
index e473a4bc,c27f80ac..fbabd999
--- a/c/compat.h
+++ b/c/compat.h
@@@ -371,8 -375,47 +376,50 @@@ static inline char *compat_mkdtemp(cha
#ifndef COMPAT_O_RDONLY
#define COMPAT_O_RDONLY O_RDONLY
#endif
+#ifndef COMPAT_O_BINARY
+#define COMPAT_O_BINARY 0
+#endif
+ /* --- coli_stdin_readable: "c'e' input su stdin adesso?", senza bloccare ---
+ *
+ * I serve loop di inkling.c e kimi_k3.c usavano select() su fd_set direttamente.
+ * Su Windows quei simboli non esistono in quella forma e le due build FALLIVANO
+ * (misurato: Linux ok, macOS ok, Windows/UCRT64 no) -- ed e' il motivo per cui
+ * le release binarie hanno sempre contenuto il solo motore GLM.
+ *
+ * La logica Windows non e' una traduzione meccanica: ha assorbito due bug.
+ * #139 select() su un handle di pipe finisce in winsock e ritorna sempre
+ * SOCKET_ERROR, quindi il loop non accettava mai una richiesta.
+ * #195 le pipe anonime NON sono oggetti attendibili: WaitForSingleObject su
+ * di esse e' undefined, e PeekNamedPipe fallisce su handle di file o
+ * console. Su stdin non-pipe si riporta "niente da leggere" invece di
+ * bloccare il loop.
+ * Duplicarla una terza volta avrebbe rifatto entrare quei due bug in due motori
+ * dove nessuno li avrebbe cercati: sta qui una volta sola.
+ *
+ * static inline: e' un header condiviso, e un TU che non la usa non deve pagarla. */
+ #ifndef _WIN32
+ #include <sys/select.h> /* select(), fd_set, struct timeval */
+ #endif
+
+ #ifdef _WIN32
+ static inline int coli_stdin_readable(void)
+ {
+ HANDLE ih = (HANDLE)_get_osfhandle(_fileno(stdin));
+ DWORD avail = 0;
+ if (ih == INVALID_HANDLE_VALUE) return 0;
+ if (PeekNamedPipe(ih, NULL, 0, NULL, &avail, NULL)) return avail > 0;
+ return 0; /* console/file: nessun poll non bloccante, meglio "niente" che bloccare */
+ }
+ #else
+ static inline int coli_stdin_readable(void)
+ {
+ /* fd 0 letterale, non STDIN_FILENO: quella macro vive in <unistd.h>, che questo
+ * header non include su tutte le piattaforme, e stdin e' 0 ovunque per POSIX. */
+ fd_set r; struct timeval tv = {0, 0};
+ FD_ZERO(&r); FD_SET(0, &r);
+ return select(1, &r, NULL, NULL, &tv) > 0 && FD_ISSET(0, &r);
+ }
+ #endif
+
#endif /* COMPAT_H */
diff --cc c/olmoe.c
index d9460d40,bb7ac1b8..9dd09110
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@@ -456,11 -492,11 +492,11 @@@ static void pin_hot_experts(Model *m)
int pinned_total = 0;
for (int l = 0; l < c->n_layers; l++) {
uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts;
--
++
uint64_t layer_total = 0;
for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e];
if (layer_total == 0) continue;
--
++
int max_pin = m->cache[l].cap - 8;
if (max_pin < 4) max_pin = 4;
@@@ -719,6 -755,19 +755,19 @@@ static void pilot_realload(Model *m, in
pthread_mutex_unlock(&g_pilot_mx);
return; /* all pinned/in-flight, skip */
}
-
++
+ /* LFRU eviction guard: don't displace a warm resident expert with a speculation */
+ if (g_pilot_evict_guard && m->freq && m->last_access && lc->slots[lru].eid >= 0) {
+ int vid = lc->slots[lru].eid;
+ uint64_t vs = lfru_score(m->freq[layer * c->n_experts + vid], m->last_access[layer * c->n_experts + vid], m->clock);
+ uint64_t cs = lfru_score(m->freq[layer * c->n_experts + eid], m->last_access[layer * c->n_experts + eid], m->clock);
+ if (cs <= vs + (vs >> 2) + (4u << 8)) {
+ m->is_queued[layer * c->n_experts + eid] = 0;
+ pthread_mutex_unlock(&g_pilot_mx);
+ return; /* drop speculation */
+ }
+ }
-
++
s = &lc->slots[lru]; s->pinned = 0;
}
s->eid = -1; s->used = ++m->clock;
diff --cc c/tests/test_cli_output.py
index 77fc86a0,7cf82db4..8da89fb8
--- a/c/tests/test_cli_output.py
+++ b/c/tests/test_cli_output.py
@@@ -15,6 -20,6 +20,7 @@@ class CliOutputLanguageTest(unittest.Te
[sys.executable, str(CLI), *args],
cwd=HERE,
text=True,
++ encoding="utf-8",
capture_output=True,
check=False,
timeout=10,
diff --cc c/tok.h
index bff8730a,b700cf7d..66853f68
--- a/c/tok.h
+++ b/c/tok.h
@@@ -169,24 -149,16 +175,24 @@@ static void tok_load(Tok *T, const cha
hm_put(&T->vocab, k, (int)strlen(k), id);
T->id2str[id]=(char*)k;
}
- /* merges: "left\0right" -> rank=i */
+ /* merges: pair arrays and "left right" strings both become
+ * "left\0right" -> rank=i. */
- int mc=1; while(mc < merges->len*2) mc<<=1;
+ int mc=1; while(merges && mc < merges->len*2) mc<<=1;
hm_init(&T->merges, mc);
- for(int i=0;i<merges->len;i++){
+ if(merges) for(int i=0;i<merges->len;i++){
jval *pr=merges->kids[i];
- if(!pr||pr->t!=J_ARR||pr->len<2||!pr->kids[0]||!pr->kids[1]||
- pr->kids[0]->t!=J_STR||pr->kids[1]->t!=J_STR){
+ const char *l=NULL, *r=NULL; int ll=0, rl=0;
+ if(pr && pr->t==J_ARR && pr->len==2 && pr->kids[0] && pr->kids[1] &&
+ pr->kids[0]->t==J_STR && pr->kids[1]->t==J_STR){
+ l=pr->kids[0]->str; r=pr->kids[1]->str;
+ ll=(int)strlen(l); rl=(int)strlen(r);
+ }else if(pr && pr->t==J_STR && pr->str){
+ const char *separator=strchr(pr->str,' ');
+ if(!separator || separator==pr->str || !separator[1]){
+ fprintf(stderr,"tokenizer.json: malformed merge entry %d\n",i); exit(1); }
+ l=pr->str; ll=(int)(separator-l); r=separator+1; rl=(int)strlen(r);
+ }else{
fprintf(stderr,"tokenizer.json: malformed merge entry %d\n",i); exit(1); }
- const char *l=pr->kids[0]->str, *r=pr->kids[1]->str;
- int ll=(int)strlen(l), rl=(int)strlen(r);
char *key=malloc(ll+1+rl); memcpy(key,l,ll); key[ll]=0; memcpy(key+ll+1,r,rl);
hm_put(&T->merges, key, ll+1+rl, i);
}
@@@ -215,9 -187,11 +221,10 @@@
jval *pat=json_get(ps->kids[i],"pattern");
jval *rx=pat?json_get(pat,"Regex"):NULL;
if(rx&&rx->t==J_STR&&strstr(rx->str,"\\p{Lu}")) T->o200k=1;
+ if(rx&&rx->t==J_STR&&strstr(rx->str,"\\p{Han}")) T->kimi=1;
}
}
- /* arena/buf restano allocati: le stringhe (j_dup) sono malloc indipendenti e ci servono vive */
- (void)arena;
+ T->json_root=root;
}
/* ---------- BPE su un pezzo: byte grezzi [a,b) -> id appesi a out ---------- */