Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(dsv4): coalesce duplicate expert loads
JustVugg95e1aa34919e
diff --git a/c/Makefile b/c/Makefileindex f86fc141..69d5aab3 100644--- a/c/Makefile+++ b/c/Makefile@@ -1207,6 +1207,7 @@ tests/test_native_quant$(EXE): tests/test_native_quant.c deepseek_v4.c native_quifeq ($(COLI_V4_SUPPORTED),1)include Makefile.deepseek-v4.units+V4_HOT_TEST_OBJ = COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16_TEST.oV4_TEST_LINK_OBJS = \COLI_V4_UNIT_ST.o \COLI_V4_UNIT_CONFIG.o \@@ -1214,7 +1215,8 @@ V4_TEST_LINK_OBJS = \COLI_V4_UNIT_SPARSE_ATTENTION.o \COLI_V4_UNIT_RESOURCE_PLAN.o \COLI_V4_UNIT_PROMPT.o \- $(addsuffix .o,$(V4_TEST_UNITS)) \+ $(filter-out COLI_V4_UNIT_EXPERT_STORE.o,$(addsuffix .o,$(V4_TEST_UNITS))) \+ $(V4_HOT_TEST_OBJ) \COLI_V4_UNIT_NATIVE_QUANT.o \COLI_V4_UNIT_NATIVE_QUANT_ROWS16.o# Windows engine objects are compiled with -DCOLI_V4_GPU_TIER (the CUDA tier@@ -1227,13 +1229,15 @@ V4_TEST_EXTRA_TARGETS = COLI_V4_UNIT_GPU.o backend_loader_dsv4.oendiftests/test_deepseek_v4$(EXE): tests/test_deepseek_v4.c deepseek_v4.c deepseek_v4.h \- Makefile.deepseek-v4.units+ Makefile.deepseek-v4.units Makefile.deepseek-v4$(MAKE) -f Makefile.deepseek-v4 ARCH=$(ARCH) deepseek-v4-test-objs \COLI_V4_UNIT_ST.o COLI_V4_UNIT_CONFIG.o COLI_V4_UNIT_MATH.o COLI_V4_UNIT_SPARSE_ATTENTION.o \COLI_V4_UNIT_RESOURCE_PLAN.o COLI_V4_UNIT_PROMPT.o COLI_V4_UNIT_NATIVE_QUANT.o \COLI_V4_UNIT_NATIVE_QUANT_ROWS16.o \+ $(V4_HOT_TEST_OBJ) \$(V4_TEST_EXTRA_TARGETS)- $(CC) $(CFLAGS) $< $(V4_TEST_LINK_OBJS) -o $@ -pthread $(LDFLAGS)+ $(CC) $(CFLAGS) -DCOLI_V4_TEST_HOOKS $< $(V4_TEST_LINK_OBJS) \+ -o $@ -pthread $(LDFLAGS)V4_OWN_DIR = build/ownershipV4_OWN_CFLAGS = $(CFLAGS) -DCOLI_V4_TEST_HOOKSdiff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4index 57fa291a..c94bc297 100644--- a/c/Makefile.deepseek-v4+++ b/c/Makefile.deepseek-v4@@ -165,6 +165,7 @@ include Makefile.deepseek-v4.unitsTARGET_OBJS = $(addsuffix .o,$(V4_TARGET_UNITS))TEST_UNIT_OBJS = $(addsuffix .o,$(V4_TEST_UNITS))+V4_HOT_TEST_OBJ = COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16_TEST.oV4_OBJS = $(TARGET_OBJS) $(V4_WIN_EXTRA_OBJS) $(REGISTRY_OBJ)REGISTRY_OBJ = expert_store_registry.oV4_SERVE_TEST := tests/test_v4_serve_framing$(if $(IS_WIN),.exe,)@@ -199,6 +200,12 @@ $(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h \native_quant_fp4_rows16.h expert_store_registry.h$(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@+$(V4_HOT_TEST_OBJ): deepseek_v4.c deepseek_v4.h deepseek_v4_internal.h \+ st.h json.h compat.h tensor.h quant.h route_trace.h \+ native_quant.h native_quant_fp4_rows16.h expert_store_registry.h+ $(CC) $(CFLAGS) -DCOLI_V4_TEST_HOOKS \+ -DCOLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16 -c deepseek_v4.c -o $@+COLI_V4_UNIT_GENERATE_STATS.o: serve_codec.h$(V4_SERVE_TEST): tests/test_v4_serve_framing.c deepseek_v4.c deepseek_v4.h \@@ -230,4 +237,5 @@ test_expert_store_registry: test_expert_store_registry.c expert_store_registry.c$(CC) -O2 -Wall -Wextra test_expert_store_registry.c expert_store_registry.c -o $@deepseek-v4-clean:- rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS) $(V4_SERVE_TEST) test_expert_store_registry+ rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS) $(V4_HOT_TEST_OBJ) \+ $(V4_SERVE_TEST) test_expert_store_registrydiff --git a/c/deepseek_v4.c b/c/deepseek_v4.cindex 07f25960..724cb67d 100644--- a/c/deepseek_v4.c+++ b/c/deepseek_v4.c@@ -6965,6 +6965,7 @@ typedef struct {typedef struct {int expert;+ int loading_expert;unsigned references;uint64_t used;unsigned char *slab;@@ -6983,6 +6984,7 @@ typedef struct {unsigned active_leases;ColiExpertStoreStats stats;pthread_mutex_t mutex;+ pthread_cond_t load_ready;double disk_sec; /* cumulative wall time spent reading expert bytes from disk */double matmul_sec; /* cumulative expert-forward compute time (#890): the phase the* dashboard needs alongside disk_sec so it stops folding@@ -7283,6 +7285,7 @@ static void destroy(ColiExpertStore *store) {compat_aligned_free(state->slots[i].slab);elsefree(state->slots[i].slab);+ pthread_cond_destroy(&state->load_ready);pthread_mutex_destroy(&state->mutex);coli_st_index_close(state->index);free(state->records);@@ -7312,6 +7315,13 @@ int coli_deepseek_v4_expert_store_open(return set_error(error, error_size, "out of memory creating ExpertStore");}pthread_mutex_init(&state->mutex, NULL);+ if (pthread_cond_init(&state->load_ready, NULL) != 0) {+ pthread_mutex_destroy(&state->mutex);+ free(state);+ free(store);+ return set_error(error, error_size,+ "cannot initialize ExpertStore load condition");+ }state->layers = options->layers;state->experts_per_layer = options->experts_per_layer;if (coli_st_index_open(&state->index, options->model_dir, error, error_size) != 0)@@ -7361,8 +7371,10 @@ int coli_deepseek_v4_expert_store_open(set_error(error, error_size, "out of memory creating expert cache slots");goto fail;}- for (int i = 0; i < state->layers * state->slots_per_layer; i++)+ for (int i = 0; i < state->layers * state->slots_per_layer; i++) {state->slots[i].expert = -1;+ state->slots[i].loading_expert = -1;+ }size_t telemetry_cells =(size_t)state->layers * state->experts_per_layer;state->ehit = calloc(telemetry_cells, sizeof(*state->ehit));@@ -7384,6 +7396,7 @@ fail:free(state->ehit);free(state->eheat);coli_st_index_close(state->index);+ pthread_cond_destroy(&state->load_ready);pthread_mutex_destroy(&state->mutex);free(state);free(store);@@ -7537,6 +7550,11 @@ static uint64_t v4_direct_flock_reads;static uint64_t v4_direct_payload_bytes;static uint64_t v4_direct_fallbacks;+#ifdef COLI_V4_TEST_HOOKS+void (*coli_v4_test_expert_read_hook)(ColiExpertKey key);+void (*coli_v4_test_expert_wait_hook)(ColiExpertKey key);+#endif+static int v4_pread_full_try(int fd, void *destination, size_t length,uint64_t offset) {unsigned char *output = destination;@@ -7827,7 +7845,9 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,hot_repin_locked(policy, state, key.layer);V4ExpertSlot *slots = layer_slots(state, key.layer);- V4ExpertSlot *slot = NULL;+ V4ExpertSlot *slot;+retry_lookup:+ slot = NULL;for (int i = 0; i < state->slots_per_layer; i++) {if (slots[i].slab && slots[i].expert == key.expert) {slot = &slots[i]; slot->references++;@@ -7843,6 +7863,24 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,pthread_mutex_unlock(&state->mutex); return 0;}}+ /* Another caller may already be filling a cache slot for this expert.+ * Wait for that single loader to publish (or abandon) the slot instead+ * of issuing the same SSD read into a second slab. Different experts do+ * not match this reservation and continue to load concurrently. */+ for (int i = 0; i < state->slots_per_layer; i++) {+ if (slots[i].loading_expert == key.expert) {+#ifdef COLI_V4_TEST_HOOKS+ if (coli_v4_test_expert_wait_hook)+ coli_v4_test_expert_wait_hook(key);+#endif+ if (pthread_cond_wait(&state->load_ready, &state->mutex) != 0) {+ pthread_mutex_unlock(&state->mutex);+ memset(view, 0, sizeof(*view));+ return -1;+ }+ goto retry_lookup;+ }+ }for (int i = 0; i < state->slots_per_layer; i++)if (!slots[i].references && !slots[i].slab) { slot = &slots[i]; break; }if (!slot)@@ -7871,13 +7909,18 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,state->stats.resident_bytes += state->record_bytes;}policy->packed[hot_slot_index(state, slot)] = 0;- slot->expert = -1; slot->references = 1;+ slot->expert = -1; slot->loading_expert = key.expert;+ slot->references = 1;state->active_leases++;slot->used = ++state->clock;pthread_mutex_unlock(&state->mutex);int rep = coli_st_expert_route(key.layer, key.expert);struct timespec disk_t0;clock_gettime(CLOCK_MONOTONIC, &disk_t0);+#ifdef COLI_V4_TEST_HOOKS+ if (coli_v4_test_expert_read_hook)+ coli_v4_test_expert_read_hook(key);+#endifint read_result = v4_read_expert_record(state, record, slot, rep);struct timespec disk_t1;clock_gettime(CLOCK_MONOTONIC, &disk_t1);@@ -7887,7 +7930,8 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,unsigned char *v4_pack_buf = NULL;/* Not when the GPU tier mirrors experts: pinned experts run from their* fp4 mirror, and a rows16-repacked slab could not be uploaded. */- if (hot_is_pinned(policy, key.layer, key.expert) && !store->gpu)+ if (!read_result &&+ hot_is_pinned(policy, key.layer, key.expert) && !store->gpu)v4_pack_buf = hot_pack_slot_prepare(record, slot);pthread_mutex_lock(&state->mutex);state->disk_sec +=@@ -7895,13 +7939,16 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,(disk_t1.tv_nsec - disk_t0.tv_nsec) * 1e-9;if (read_result) {slot->references = 0; slot->expert = -1;+ slot->loading_expert = -1;if (state->active_leases) state->active_leases--;free(v4_pack_buf);+ pthread_cond_broadcast(&state->load_ready);pthread_mutex_unlock(&state->mutex);memset(view, 0, sizeof(*view));return -1;}- slot->expert = key.expert; slot->used = ++state->clock;+ slot->expert = key.expert; slot->loading_expert = -1;+ slot->used = ++state->clock;state->stats.misses++; state->stats.bytes_read += record->record_bytes;hot_pack_slot_commit(policy, state, record, slot, v4_pack_buf);v4_pack_buf = NULL;@@ -7910,6 +7957,7 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,hot_fill_view(&view->down, record, slot, V4_W2, policy, state);hot_fill_view(&view->up, record, slot, V4_W3, policy, state);view->lease = slot;+ pthread_cond_broadcast(&state->load_ready);pthread_mutex_unlock(&state->mutex); return 0;}diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.hindex de4a108d..f9a811a4 100644--- a/c/deepseek_v4_internal.h+++ b/c/deepseek_v4_internal.h@@ -954,6 +954,8 @@ int coli_st_read_at_engine(ColiV4Engine *engine,extern int coli_v4_test_fail_expert_store_open;extern int coli_v4_test_skip_expert_store_open;extern int coli_v4_test_closed_owned_index;+extern void (*coli_v4_test_expert_read_hook)(ColiExpertKey key);+extern void (*coli_v4_test_expert_wait_hook)(ColiExpertKey key);ColiV4Session *coli_v4_test_session_bare_create(ColiV4Engine *engine);void coli_v4_test_session_bare_destroy(ColiV4Session *session);diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.cindex dd782a4d..760d5334 100644--- a/c/tests/test_deepseek_v4.c+++ b/c/tests/test_deepseek_v4.c@@ -8,6 +8,7 @@#include <fcntl.h>#include <limits.h>#include <math.h>+#include <pthread.h>#include <stdint.h>#include <stdio.h>#include <stdlib.h>@@ -306,10 +307,22 @@ static int write_fixture(const char *path) {"\"resident\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[3,7]},""\"layers.0.ffn.experts.0.w1.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[7,23]},""\"layers.0.ffn.experts.0.w2.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[23,39]},"- "\"layers.0.ffn.experts.0.w3.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[39,55]}"+ "\"layers.0.ffn.experts.0.w3.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[39,55]},"+ "\"layers.0.ffn.experts.1.w1.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[55,56]},"+ "\"layers.0.ffn.experts.1.w2.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[56,57]},"+ "\"layers.0.ffn.experts.1.w3.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[57,58]},"+ "\"layers.0.ffn.experts.1.w1.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[58,74]},"+ "\"layers.0.ffn.experts.1.w2.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[74,90]},"+ "\"layers.0.ffn.experts.1.w3.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[90,106]},"+ "\"layers.0.ffn.experts.2.w1.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[106,107]},"+ "\"layers.0.ffn.experts.2.w2.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[107,108]},"+ "\"layers.0.ffn.experts.2.w3.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[108,109]},"+ "\"layers.0.ffn.experts.2.w1.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[109,125]},"+ "\"layers.0.ffn.experts.2.w2.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[125,141]},"+ "\"layers.0.ffn.experts.2.w3.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[141,157]}""}";- unsigned char payload[55];- for (int i = 0; i < 55; i++) payload[i] = (unsigned char)i;+ unsigned char payload[157];+ for (int i = 0; i < 157; i++) payload[i] = (unsigned char)i;uint64_t header_length = sizeof(header) - 1;int fd = open(path, O_CREAT | O_TRUNC | O_WRONLY | COMPAT_O_BINARY, 0600);if (fd < 0) return -1;@@ -320,16 +333,125 @@ static int write_fixture(const char *path) {return result ? -1 : 0;}+enum {+ TEST_READ_SAME_EXPERT,+ TEST_READ_DISTINCT_EXPERTS+};++static pthread_mutex_t expert_hook_mutex = PTHREAD_MUTEX_INITIALIZER;+static pthread_cond_t expert_hook_ready = PTHREAD_COND_INITIALIZER;+static int expert_hook_mode;+static int expert_hook_reads;+static int expert_hook_waits;+static int expert_hook_active;+static int expert_hook_max_active;+static int expert_hook_abort;++static void reset_expert_hooks(int mode) {+ pthread_mutex_lock(&expert_hook_mutex);+ expert_hook_mode = mode;+ expert_hook_reads = 0;+ expert_hook_waits = 0;+ expert_hook_active = 0;+ expert_hook_max_active = 0;+ expert_hook_abort = 0;+ pthread_mutex_unlock(&expert_hook_mutex);+}++static void abort_expert_hooks(void) {+ pthread_mutex_lock(&expert_hook_mutex);+ expert_hook_abort = 1;+ pthread_cond_broadcast(&expert_hook_ready);+ pthread_mutex_unlock(&expert_hook_mutex);+}++static void test_expert_read_hook(ColiExpertKey key) {+ (void)key;+ pthread_mutex_lock(&expert_hook_mutex);+ expert_hook_reads++;+ expert_hook_active++;+ if (expert_hook_active > expert_hook_max_active)+ expert_hook_max_active = expert_hook_active;+ pthread_cond_broadcast(&expert_hook_ready);+ if (expert_hook_mode == TEST_READ_SAME_EXPERT) {+ while (!expert_hook_abort && !expert_hook_waits &&+ expert_hook_reads < 2)+ pthread_cond_wait(&expert_hook_ready, &expert_hook_mutex);+ } else {+ /* A wait here means distinct experts were incorrectly coalesced. */+ while (!expert_hook_abort && expert_hook_reads < 2 &&+ !expert_hook_waits)+ pthread_cond_wait(&expert_hook_ready, &expert_hook_mutex);+ }+ expert_hook_active--;+ pthread_cond_broadcast(&expert_hook_ready);+ pthread_mutex_unlock(&expert_hook_mutex);+}++static void test_expert_wait_hook(ColiExpertKey key) {+ (void)key;+ pthread_mutex_lock(&expert_hook_mutex);+ expert_hook_waits++;+ pthread_cond_broadcast(&expert_hook_ready);+ pthread_mutex_unlock(&expert_hook_mutex);+}++typedef struct {+ ColiExpertStore *store;+ ColiExpertKey key;+ ColiExpertView view;+ int result;+} ExpertLookupJob;++static void *expert_lookup_worker(void *opaque) {+ ExpertLookupJob *job = opaque;+ job->result = coli_expert_lookup(job->store, job->key, &job->view);+ return NULL;+}++static int run_parallel_lookups(ColiExpertStore *store,+ ColiExpertKey first, ColiExpertKey second,+ ExpertLookupJob jobs[2]) {+ pthread_t threads[2];+ memset(jobs, 0, 2 * sizeof(*jobs));+ jobs[0].store = jobs[1].store = store;+ jobs[0].key = first;+ jobs[1].key = second;+ coli_v4_test_expert_read_hook = test_expert_read_hook;+ coli_v4_test_expert_wait_hook = test_expert_wait_hook;+ if (pthread_create(&threads[0], NULL, expert_lookup_worker, &jobs[0])) {+ coli_v4_test_expert_read_hook = NULL;+ coli_v4_test_expert_wait_hook = NULL;+ return -1;+ }+ if (pthread_create(&threads[1], NULL, expert_lookup_worker, &jobs[1])) {+ abort_expert_hooks();+ pthread_join(threads[0], NULL);+ if (!jobs[0].result) coli_expert_release(store, &jobs[0].view);+ coli_v4_test_expert_read_hook = NULL;+ coli_v4_test_expert_wait_hook = NULL;+ return -1;+ }+ int join_result = pthread_join(threads[0], NULL) |+ pthread_join(threads[1], NULL);+ coli_v4_test_expert_read_hook = NULL;+ coli_v4_test_expert_wait_hook = NULL;+ return join_result || jobs[0].result || jobs[1].result ? -1 : 0;+}+static int test_expert_store(void) {/* Native MinGW binaries do not resolve the MSYS /tmp mount. */char directory[] = "colibri-v4-store-XXXXXX";char path[256], error[256];+ setenv("COLI_V4_AUTOPIN", "0", 1);+ setenv("COLI_V4_SAVE_USAGE", "0", 1);+ setenv("COLI_V4_ROWS16", "0", 1);if (!mkdtemp(directory)) { perror("mkdtemp"); return 1; }snprintf(path, sizeof(path), "%s/model.safetensors", directory);if (write_fixture(path) != 0) { perror("write_fixture"); return 1; }ColiDeepSeekV4ExpertStoreOptions options = {- directory, 1, 1, 51, -1, 0+ directory, 1, 3, 153, -1, 0};ColiExpertStore *store = NULL;if (coli_deepseek_v4_expert_store_open(&options, &store,@@ -342,36 +464,46 @@ static int test_expert_store(void) {if (store->ops->prefetch(store, &key, 1) != 1) {fprintf(stderr, "prefetch failed\n"); return 1;}- if (coli_expert_lookup(store, key, &view) != 0) {- fprintf(stderr, "lookup failed\n"); return 1;+ ExpertLookupJob same[2];+ reset_expert_hooks(TEST_READ_SAME_EXPERT);+ if (run_parallel_lookups(store, key, key, same) != 0) {+ fprintf(stderr, "parallel same-expert lookup failed\n"); return 1;+ }+ if (expert_hook_reads != 1 || expert_hook_waits < 1 ||+ expert_hook_max_active != 1) {+ fprintf(stderr,+ "same-expert load was not single-flight: reads=%d waits=%d active=%d\n",+ expert_hook_reads, expert_hook_waits,+ expert_hook_max_active);+ return 1;}- if (view.gate.format != COLI_TENSOR_FP4_NATIVE_BLOCK ||- view.gate.rows != 1 || view.gate.columns != 32 ||- view.gate.data_bytes != 16 || view.gate.scale_bytes != 1 ||- ((const unsigned char *)view.gate.scales)[0] != 0 ||- ((const unsigned char *)view.gate.data)[0] != 7 ||- ((const unsigned char *)view.down.data)[0] != 23 ||- ((const unsigned char *)view.up.data)[0] != 39)+ ColiExpertView *loaded = &same[0].view;+ if (loaded->gate.format != COLI_TENSOR_FP4_NATIVE_BLOCK ||+ loaded->gate.rows != 1 || loaded->gate.columns != 32 ||+ loaded->gate.data_bytes != 16 || loaded->gate.scale_bytes != 1 ||+ ((const unsigned char *)loaded->gate.scales)[0] != 0 ||+ ((const unsigned char *)loaded->gate.data)[0] != 7 ||+ ((const unsigned char *)loaded->down.data)[0] != 23 ||+ ((const unsigned char *)loaded->up.data)[0] != 39){ fprintf(stderr, "expert view mismatch: format=%d rows=%lld columns=%lld data=%zu scales=%zu bytes=%u/%u/%u/%u\n",- (int)view.gate.format, (long long)view.gate.rows,- (long long)view.gate.columns, view.gate.data_bytes,- view.gate.scale_bytes,- ((const unsigned char *)view.gate.scales)[0],- ((const unsigned char *)view.gate.data)[0],- ((const unsigned char *)view.down.data)[0],- ((const unsigned char *)view.up.data)[0]); return 1; }- coli_expert_release(store, &view);+ (int)loaded->gate.format, (long long)loaded->gate.rows,+ (long long)loaded->gate.columns, loaded->gate.data_bytes,+ loaded->gate.scale_bytes,+ ((const unsigned char *)loaded->gate.scales)[0],+ ((const unsigned char *)loaded->gate.data)[0],+ ((const unsigned char *)loaded->down.data)[0],+ ((const unsigned char *)loaded->up.data)[0]); return 1; }+ coli_expert_release(store, &same[0].view);{static const ColiExpertView zero;- if (memcmp(&view, &zero, sizeof(view)) != 0) {+ if (memcmp(&same[0].view, &zero, sizeof(same[0].view)) != 0) {fprintf(stderr, "release did not clear view\n");return 1;}}/* Double release of a cleared view is a no-op. */- coli_expert_release(store, &view);- if (coli_expert_lookup(store, key, &view) != 0) return 1;- coli_expert_release(store, &view);+ coli_expert_release(store, &same[0].view);+ coli_expert_release(store, &same[1].view);/* Invalid key lookup must fail and clear the view. */memset(&view, 0x3c, sizeof(view));if (coli_expert_lookup(store, (ColiExpertKey){9, 9}, &view) == 0) return 1;@@ -386,7 +518,7 @@ static int test_expert_store(void) {store->ops->stats(store, &stats);if (stats.requests != 2 || stats.hits != 1 || stats.misses != 1 ||stats.prefetched != 1 || stats.bytes_read != 51 ||- stats.resident_bytes != 51 || stats.capacity_bytes != 51)+ stats.resident_bytes != 51 || stats.capacity_bytes != 153){ fprintf(stderr, "expert stats mismatch: requests=%llu hits=%llu misses=%llu prefetched=%llu bytes=%llu resident=%llu capacity=%llu\n",(unsigned long long)stats.requests,(unsigned long long)stats.hits,@@ -395,6 +527,37 @@ static int test_expert_store(void) {(unsigned long long)stats.bytes_read,(unsigned long long)stats.resident_bytes,(unsigned long long)stats.capacity_bytes); return 1; }++ ExpertLookupJob distinct[2];+ reset_expert_hooks(TEST_READ_DISTINCT_EXPERTS);+ if (run_parallel_lookups(store, (ColiExpertKey){0, 1},+ (ColiExpertKey){0, 2}, distinct) != 0) {+ fprintf(stderr, "parallel distinct-expert lookup failed\n"); return 1;+ }+ if (expert_hook_reads != 2 || expert_hook_waits != 0 ||+ expert_hook_max_active != 2) {+ fprintf(stderr,+ "distinct expert loads did not overlap: reads=%d waits=%d active=%d\n",+ expert_hook_reads, expert_hook_waits,+ expert_hook_max_active);+ return 1;+ }+ coli_expert_release(store, &distinct[0].view);+ coli_expert_release(store, &distinct[1].view);+ store->ops->stats(store, &stats);+ if (stats.requests != 4 || stats.hits != 1 || stats.misses != 3 ||+ stats.prefetched != 1 || stats.bytes_read != 153 ||+ stats.resident_bytes != 153 || stats.capacity_bytes != 153) {+ fprintf(stderr,+ "concurrent expert stats mismatch: requests=%llu hits=%llu misses=%llu bytes=%llu resident=%llu capacity=%llu\n",+ (unsigned long long)stats.requests,+ (unsigned long long)stats.hits,+ (unsigned long long)stats.misses,+ (unsigned long long)stats.bytes_read,+ (unsigned long long)stats.resident_bytes,+ (unsigned long long)stats.capacity_bytes);+ return 1;+ }store->ops->destroy(store);unlink(path);rmdir(directory);