Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(dsv4): coalesce duplicate expert loads

JustVugg95e1aa34919e
diff --git a/c/Makefile b/c/Makefile
index f86fc141..69d5aab3 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1207,6 +1207,7 @@ tests/test_native_quant$(EXE): tests/test_native_quant.c deepseek_v4.c native_qu
ifeq ($(COLI_V4_SUPPORTED),1)
include Makefile.deepseek-v4.units
+V4_HOT_TEST_OBJ = COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16_TEST.o
V4_TEST_LINK_OBJS = \
COLI_V4_UNIT_ST.o \
COLI_V4_UNIT_CONFIG.o \
@@ -1214,7 +1215,8 @@ V4_TEST_LINK_OBJS = \
COLI_V4_UNIT_SPARSE_ATTENTION.o \
COLI_V4_UNIT_RESOURCE_PLAN.o \
COLI_V4_UNIT_PROMPT.o \
- $(addsuffix .o,$(V4_TEST_UNITS)) \
+ $(filter-out COLI_V4_UNIT_EXPERT_STORE.o,$(addsuffix .o,$(V4_TEST_UNITS))) \
+ $(V4_HOT_TEST_OBJ) \
COLI_V4_UNIT_NATIVE_QUANT.o \
COLI_V4_UNIT_NATIVE_QUANT_ROWS16.o
# Windows engine objects are compiled with -DCOLI_V4_GPU_TIER (the CUDA tier
@@ -1227,13 +1229,15 @@ V4_TEST_EXTRA_TARGETS = COLI_V4_UNIT_GPU.o backend_loader_dsv4.o
endif
tests/test_deepseek_v4$(EXE): tests/test_deepseek_v4.c deepseek_v4.c deepseek_v4.h \
- Makefile.deepseek-v4.units
+ Makefile.deepseek-v4.units Makefile.deepseek-v4
$(MAKE) -f Makefile.deepseek-v4 ARCH=$(ARCH) deepseek-v4-test-objs \
COLI_V4_UNIT_ST.o COLI_V4_UNIT_CONFIG.o COLI_V4_UNIT_MATH.o COLI_V4_UNIT_SPARSE_ATTENTION.o \
COLI_V4_UNIT_RESOURCE_PLAN.o COLI_V4_UNIT_PROMPT.o COLI_V4_UNIT_NATIVE_QUANT.o \
COLI_V4_UNIT_NATIVE_QUANT_ROWS16.o \
+ $(V4_HOT_TEST_OBJ) \
$(V4_TEST_EXTRA_TARGETS)
- $(CC) $(CFLAGS) $< $(V4_TEST_LINK_OBJS) -o $@ -pthread $(LDFLAGS)
+ $(CC) $(CFLAGS) -DCOLI_V4_TEST_HOOKS $< $(V4_TEST_LINK_OBJS) \
+ -o $@ -pthread $(LDFLAGS)
V4_OWN_DIR = build/ownership
V4_OWN_CFLAGS = $(CFLAGS) -DCOLI_V4_TEST_HOOKS
diff --git a/c/Makefile.deepseek-v4 b/c/Makefile.deepseek-v4
index 57fa291a..c94bc297 100644
--- a/c/Makefile.deepseek-v4
+++ b/c/Makefile.deepseek-v4
@@ -165,6 +165,7 @@ include Makefile.deepseek-v4.units
TARGET_OBJS = $(addsuffix .o,$(V4_TARGET_UNITS))
TEST_UNIT_OBJS = $(addsuffix .o,$(V4_TEST_UNITS))
+V4_HOT_TEST_OBJ = COLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16_TEST.o
V4_OBJS = $(TARGET_OBJS) $(V4_WIN_EXTRA_OBJS) $(REGISTRY_OBJ)
REGISTRY_OBJ = expert_store_registry.o
V4_SERVE_TEST := tests/test_v4_serve_framing$(if $(IS_WIN),.exe,)
@@ -199,6 +200,12 @@ $(TARGET_OBJS) $(TEST_UNIT_OBJS): %.o: deepseek_v4.c deepseek_v4.h \
native_quant_fp4_rows16.h expert_store_registry.h
$(CC) $(CFLAGS) -D$* -c deepseek_v4.c -o $@
+$(V4_HOT_TEST_OBJ): deepseek_v4.c deepseek_v4.h deepseek_v4_internal.h \
+ st.h json.h compat.h tensor.h quant.h route_trace.h \
+ native_quant.h native_quant_fp4_rows16.h expert_store_registry.h
+ $(CC) $(CFLAGS) -DCOLI_V4_TEST_HOOKS \
+ -DCOLI_V4_UNIT_EXPERT_STORE_HOT_ROWS16 -c deepseek_v4.c -o $@
+
COLI_V4_UNIT_GENERATE_STATS.o: serve_codec.h
$(V4_SERVE_TEST): tests/test_v4_serve_framing.c deepseek_v4.c deepseek_v4.h \
@@ -230,4 +237,5 @@ test_expert_store_registry: test_expert_store_registry.c expert_store_registry.c
$(CC) -O2 -Wall -Wextra test_expert_store_registry.c expert_store_registry.c -o $@
deepseek-v4-clean:
- rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS) $(V4_SERVE_TEST) test_expert_store_registry
+ rm -f $(V4_BINARY) $(V4_OBJS) $(TEST_UNIT_OBJS) $(V4_HOT_TEST_OBJ) \
+ $(V4_SERVE_TEST) test_expert_store_registry
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 07f25960..724cb67d 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -6965,6 +6965,7 @@ typedef struct {
typedef struct {
int expert;
+ int loading_expert;
unsigned references;
uint64_t used;
unsigned char *slab;
@@ -6983,6 +6984,7 @@ typedef struct {
unsigned active_leases;
ColiExpertStoreStats stats;
pthread_mutex_t mutex;
+ pthread_cond_t load_ready;
double disk_sec; /* cumulative wall time spent reading expert bytes from disk */
double matmul_sec; /* cumulative expert-forward compute time (#890): the phase the
* dashboard needs alongside disk_sec so it stops folding
@@ -7283,6 +7285,7 @@ static void destroy(ColiExpertStore *store) {
compat_aligned_free(state->slots[i].slab);
else
free(state->slots[i].slab);
+ pthread_cond_destroy(&state->load_ready);
pthread_mutex_destroy(&state->mutex);
coli_st_index_close(state->index);
free(state->records);
@@ -7312,6 +7315,13 @@ int coli_deepseek_v4_expert_store_open(
return set_error(error, error_size, "out of memory creating ExpertStore");
}
pthread_mutex_init(&state->mutex, NULL);
+ if (pthread_cond_init(&state->load_ready, NULL) != 0) {
+ pthread_mutex_destroy(&state->mutex);
+ free(state);
+ free(store);
+ return set_error(error, error_size,
+ "cannot initialize ExpertStore load condition");
+ }
state->layers = options->layers;
state->experts_per_layer = options->experts_per_layer;
if (coli_st_index_open(&state->index, options->model_dir, error, error_size) != 0)
@@ -7361,8 +7371,10 @@ int coli_deepseek_v4_expert_store_open(
set_error(error, error_size, "out of memory creating expert cache slots");
goto fail;
}
- for (int i = 0; i < state->layers * state->slots_per_layer; i++)
+ for (int i = 0; i < state->layers * state->slots_per_layer; i++) {
state->slots[i].expert = -1;
+ state->slots[i].loading_expert = -1;
+ }
size_t telemetry_cells =
(size_t)state->layers * state->experts_per_layer;
state->ehit = calloc(telemetry_cells, sizeof(*state->ehit));
@@ -7384,6 +7396,7 @@ fail:
free(state->ehit);
free(state->eheat);
coli_st_index_close(state->index);
+ pthread_cond_destroy(&state->load_ready);
pthread_mutex_destroy(&state->mutex);
free(state);
free(store);
@@ -7537,6 +7550,11 @@ static uint64_t v4_direct_flock_reads;
static uint64_t v4_direct_payload_bytes;
static uint64_t v4_direct_fallbacks;
+#ifdef COLI_V4_TEST_HOOKS
+void (*coli_v4_test_expert_read_hook)(ColiExpertKey key);
+void (*coli_v4_test_expert_wait_hook)(ColiExpertKey key);
+#endif
+
static int v4_pread_full_try(int fd, void *destination, size_t length,
uint64_t offset) {
unsigned char *output = destination;
@@ -7827,7 +7845,9 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
hot_repin_locked(policy, state, key.layer);
V4ExpertSlot *slots = layer_slots(state, key.layer);
- V4ExpertSlot *slot = NULL;
+ V4ExpertSlot *slot;
+retry_lookup:
+ slot = NULL;
for (int i = 0; i < state->slots_per_layer; i++) {
if (slots[i].slab && slots[i].expert == key.expert) {
slot = &slots[i]; slot->references++;
@@ -7843,6 +7863,24 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
pthread_mutex_unlock(&state->mutex); return 0;
}
}
+ /* Another caller may already be filling a cache slot for this expert.
+ * Wait for that single loader to publish (or abandon) the slot instead
+ * of issuing the same SSD read into a second slab. Different experts do
+ * not match this reservation and continue to load concurrently. */
+ for (int i = 0; i < state->slots_per_layer; i++) {
+ if (slots[i].loading_expert == key.expert) {
+#ifdef COLI_V4_TEST_HOOKS
+ if (coli_v4_test_expert_wait_hook)
+ coli_v4_test_expert_wait_hook(key);
+#endif
+ if (pthread_cond_wait(&state->load_ready, &state->mutex) != 0) {
+ pthread_mutex_unlock(&state->mutex);
+ memset(view, 0, sizeof(*view));
+ return -1;
+ }
+ goto retry_lookup;
+ }
+ }
for (int i = 0; i < state->slots_per_layer; i++)
if (!slots[i].references && !slots[i].slab) { slot = &slots[i]; break; }
if (!slot)
@@ -7871,13 +7909,18 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
state->stats.resident_bytes += state->record_bytes;
}
policy->packed[hot_slot_index(state, slot)] = 0;
- slot->expert = -1; slot->references = 1;
+ slot->expert = -1; slot->loading_expert = key.expert;
+ slot->references = 1;
state->active_leases++;
slot->used = ++state->clock;
pthread_mutex_unlock(&state->mutex);
int rep = coli_st_expert_route(key.layer, key.expert);
struct timespec disk_t0;
clock_gettime(CLOCK_MONOTONIC, &disk_t0);
+#ifdef COLI_V4_TEST_HOOKS
+ if (coli_v4_test_expert_read_hook)
+ coli_v4_test_expert_read_hook(key);
+#endif
int read_result = v4_read_expert_record(state, record, slot, rep);
struct timespec disk_t1;
clock_gettime(CLOCK_MONOTONIC, &disk_t1);
@@ -7887,7 +7930,8 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
unsigned char *v4_pack_buf = NULL;
/* Not when the GPU tier mirrors experts: pinned experts run from their
* fp4 mirror, and a rows16-repacked slab could not be uploaded. */
- if (hot_is_pinned(policy, key.layer, key.expert) && !store->gpu)
+ if (!read_result &&
+ hot_is_pinned(policy, key.layer, key.expert) && !store->gpu)
v4_pack_buf = hot_pack_slot_prepare(record, slot);
pthread_mutex_lock(&state->mutex);
state->disk_sec +=
@@ -7895,13 +7939,16 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
(disk_t1.tv_nsec - disk_t0.tv_nsec) * 1e-9;
if (read_result) {
slot->references = 0; slot->expert = -1;
+ slot->loading_expert = -1;
if (state->active_leases) state->active_leases--;
free(v4_pack_buf);
+ pthread_cond_broadcast(&state->load_ready);
pthread_mutex_unlock(&state->mutex);
memset(view, 0, sizeof(*view));
return -1;
}
- slot->expert = key.expert; slot->used = ++state->clock;
+ slot->expert = key.expert; slot->loading_expert = -1;
+ slot->used = ++state->clock;
state->stats.misses++; state->stats.bytes_read += record->record_bytes;
hot_pack_slot_commit(policy, state, record, slot, v4_pack_buf);
v4_pack_buf = NULL;
@@ -7910,6 +7957,7 @@ static int lookup_hot(ColiExpertStore *store, ColiExpertKey key,
hot_fill_view(&view->down, record, slot, V4_W2, policy, state);
hot_fill_view(&view->up, record, slot, V4_W3, policy, state);
view->lease = slot;
+ pthread_cond_broadcast(&state->load_ready);
pthread_mutex_unlock(&state->mutex); return 0;
}
diff --git a/c/deepseek_v4_internal.h b/c/deepseek_v4_internal.h
index de4a108d..f9a811a4 100644
--- a/c/deepseek_v4_internal.h
+++ b/c/deepseek_v4_internal.h
@@ -954,6 +954,8 @@ int coli_st_read_at_engine(ColiV4Engine *engine,
extern int coli_v4_test_fail_expert_store_open;
extern int coli_v4_test_skip_expert_store_open;
extern int coli_v4_test_closed_owned_index;
+extern void (*coli_v4_test_expert_read_hook)(ColiExpertKey key);
+extern void (*coli_v4_test_expert_wait_hook)(ColiExpertKey key);
ColiV4Session *coli_v4_test_session_bare_create(ColiV4Engine *engine);
void coli_v4_test_session_bare_destroy(ColiV4Session *session);
diff --git a/c/tests/test_deepseek_v4.c b/c/tests/test_deepseek_v4.c
index dd782a4d..760d5334 100644
--- a/c/tests/test_deepseek_v4.c
+++ b/c/tests/test_deepseek_v4.c
@@ -8,6 +8,7 @@
#include <fcntl.h>
#include <limits.h>
#include <math.h>
+#include <pthread.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
@@ -306,10 +307,22 @@ static int write_fixture(const char *path) {
"\"resident\":{\"dtype\":\"F32\",\"shape\":[1],\"data_offsets\":[3,7]},"
"\"layers.0.ffn.experts.0.w1.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[7,23]},"
"\"layers.0.ffn.experts.0.w2.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[23,39]},"
- "\"layers.0.ffn.experts.0.w3.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[39,55]}"
+ "\"layers.0.ffn.experts.0.w3.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[39,55]},"
+ "\"layers.0.ffn.experts.1.w1.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[55,56]},"
+ "\"layers.0.ffn.experts.1.w2.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[56,57]},"
+ "\"layers.0.ffn.experts.1.w3.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[57,58]},"
+ "\"layers.0.ffn.experts.1.w1.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[58,74]},"
+ "\"layers.0.ffn.experts.1.w2.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[74,90]},"
+ "\"layers.0.ffn.experts.1.w3.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[90,106]},"
+ "\"layers.0.ffn.experts.2.w1.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[106,107]},"
+ "\"layers.0.ffn.experts.2.w2.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[107,108]},"
+ "\"layers.0.ffn.experts.2.w3.scale\":{\"dtype\":\"F8_E8M0\",\"shape\":[1,1],\"data_offsets\":[108,109]},"
+ "\"layers.0.ffn.experts.2.w1.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[109,125]},"
+ "\"layers.0.ffn.experts.2.w2.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[125,141]},"
+ "\"layers.0.ffn.experts.2.w3.weight\":{\"dtype\":\"I8\",\"shape\":[1,16],\"data_offsets\":[141,157]}"
"}";
- unsigned char payload[55];
- for (int i = 0; i < 55; i++) payload[i] = (unsigned char)i;
+ unsigned char payload[157];
+ for (int i = 0; i < 157; i++) payload[i] = (unsigned char)i;
uint64_t header_length = sizeof(header) - 1;
int fd = open(path, O_CREAT | O_TRUNC | O_WRONLY | COMPAT_O_BINARY, 0600);
if (fd < 0) return -1;
@@ -320,16 +333,125 @@ static int write_fixture(const char *path) {
return result ? -1 : 0;
}
+enum {
+ TEST_READ_SAME_EXPERT,
+ TEST_READ_DISTINCT_EXPERTS
+};
+
+static pthread_mutex_t expert_hook_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t expert_hook_ready = PTHREAD_COND_INITIALIZER;
+static int expert_hook_mode;
+static int expert_hook_reads;
+static int expert_hook_waits;
+static int expert_hook_active;
+static int expert_hook_max_active;
+static int expert_hook_abort;
+
+static void reset_expert_hooks(int mode) {
+ pthread_mutex_lock(&expert_hook_mutex);
+ expert_hook_mode = mode;
+ expert_hook_reads = 0;
+ expert_hook_waits = 0;
+ expert_hook_active = 0;
+ expert_hook_max_active = 0;
+ expert_hook_abort = 0;
+ pthread_mutex_unlock(&expert_hook_mutex);
+}
+
+static void abort_expert_hooks(void) {
+ pthread_mutex_lock(&expert_hook_mutex);
+ expert_hook_abort = 1;
+ pthread_cond_broadcast(&expert_hook_ready);
+ pthread_mutex_unlock(&expert_hook_mutex);
+}
+
+static void test_expert_read_hook(ColiExpertKey key) {
+ (void)key;
+ pthread_mutex_lock(&expert_hook_mutex);
+ expert_hook_reads++;
+ expert_hook_active++;
+ if (expert_hook_active > expert_hook_max_active)
+ expert_hook_max_active = expert_hook_active;
+ pthread_cond_broadcast(&expert_hook_ready);
+ if (expert_hook_mode == TEST_READ_SAME_EXPERT) {
+ while (!expert_hook_abort && !expert_hook_waits &&
+ expert_hook_reads < 2)
+ pthread_cond_wait(&expert_hook_ready, &expert_hook_mutex);
+ } else {
+ /* A wait here means distinct experts were incorrectly coalesced. */
+ while (!expert_hook_abort && expert_hook_reads < 2 &&
+ !expert_hook_waits)
+ pthread_cond_wait(&expert_hook_ready, &expert_hook_mutex);
+ }
+ expert_hook_active--;
+ pthread_cond_broadcast(&expert_hook_ready);
+ pthread_mutex_unlock(&expert_hook_mutex);
+}
+
+static void test_expert_wait_hook(ColiExpertKey key) {
+ (void)key;
+ pthread_mutex_lock(&expert_hook_mutex);
+ expert_hook_waits++;
+ pthread_cond_broadcast(&expert_hook_ready);
+ pthread_mutex_unlock(&expert_hook_mutex);
+}
+
+typedef struct {
+ ColiExpertStore *store;
+ ColiExpertKey key;
+ ColiExpertView view;
+ int result;
+} ExpertLookupJob;
+
+static void *expert_lookup_worker(void *opaque) {
+ ExpertLookupJob *job = opaque;
+ job->result = coli_expert_lookup(job->store, job->key, &job->view);
+ return NULL;
+}
+
+static int run_parallel_lookups(ColiExpertStore *store,
+ ColiExpertKey first, ColiExpertKey second,
+ ExpertLookupJob jobs[2]) {
+ pthread_t threads[2];
+ memset(jobs, 0, 2 * sizeof(*jobs));
+ jobs[0].store = jobs[1].store = store;
+ jobs[0].key = first;
+ jobs[1].key = second;
+ coli_v4_test_expert_read_hook = test_expert_read_hook;
+ coli_v4_test_expert_wait_hook = test_expert_wait_hook;
+ if (pthread_create(&threads[0], NULL, expert_lookup_worker, &jobs[0])) {
+ coli_v4_test_expert_read_hook = NULL;
+ coli_v4_test_expert_wait_hook = NULL;
+ return -1;
+ }
+ if (pthread_create(&threads[1], NULL, expert_lookup_worker, &jobs[1])) {
+ abort_expert_hooks();
+ pthread_join(threads[0], NULL);
+ if (!jobs[0].result) coli_expert_release(store, &jobs[0].view);
+ coli_v4_test_expert_read_hook = NULL;
+ coli_v4_test_expert_wait_hook = NULL;
+ return -1;
+ }
+ int join_result = pthread_join(threads[0], NULL) |
+ pthread_join(threads[1], NULL);
+ coli_v4_test_expert_read_hook = NULL;
+ coli_v4_test_expert_wait_hook = NULL;
+ return join_result || jobs[0].result || jobs[1].result ? -1 : 0;
+}
+
static int test_expert_store(void) {
/* Native MinGW binaries do not resolve the MSYS /tmp mount. */
char directory[] = "colibri-v4-store-XXXXXX";
char path[256], error[256];
+ setenv("COLI_V4_AUTOPIN", "0", 1);
+ setenv("COLI_V4_SAVE_USAGE", "0", 1);
+ setenv("COLI_V4_ROWS16", "0", 1);
if (!mkdtemp(directory)) { perror("mkdtemp"); return 1; }
snprintf(path, sizeof(path), "%s/model.safetensors", directory);
if (write_fixture(path) != 0) { perror("write_fixture"); return 1; }
ColiDeepSeekV4ExpertStoreOptions options = {
- directory, 1, 1, 51, -1, 0
+ directory, 1, 3, 153, -1, 0
};
ColiExpertStore *store = NULL;
if (coli_deepseek_v4_expert_store_open(&options, &store,
@@ -342,36 +464,46 @@ static int test_expert_store(void) {
if (store->ops->prefetch(store, &key, 1) != 1) {
fprintf(stderr, "prefetch failed\n"); return 1;
}
- if (coli_expert_lookup(store, key, &view) != 0) {
- fprintf(stderr, "lookup failed\n"); return 1;
+ ExpertLookupJob same[2];
+ reset_expert_hooks(TEST_READ_SAME_EXPERT);
+ if (run_parallel_lookups(store, key, key, same) != 0) {
+ fprintf(stderr, "parallel same-expert lookup failed\n"); return 1;
+ }
+ if (expert_hook_reads != 1 || expert_hook_waits < 1 ||
+ expert_hook_max_active != 1) {
+ fprintf(stderr,
+ "same-expert load was not single-flight: reads=%d waits=%d active=%d\n",
+ expert_hook_reads, expert_hook_waits,
+ expert_hook_max_active);
+ return 1;
}
- if (view.gate.format != COLI_TENSOR_FP4_NATIVE_BLOCK ||
- view.gate.rows != 1 || view.gate.columns != 32 ||
- view.gate.data_bytes != 16 || view.gate.scale_bytes != 1 ||
- ((const unsigned char *)view.gate.scales)[0] != 0 ||
- ((const unsigned char *)view.gate.data)[0] != 7 ||
- ((const unsigned char *)view.down.data)[0] != 23 ||
- ((const unsigned char *)view.up.data)[0] != 39)
+ ColiExpertView *loaded = &same[0].view;
+ if (loaded->gate.format != COLI_TENSOR_FP4_NATIVE_BLOCK ||
+ loaded->gate.rows != 1 || loaded->gate.columns != 32 ||
+ loaded->gate.data_bytes != 16 || loaded->gate.scale_bytes != 1 ||
+ ((const unsigned char *)loaded->gate.scales)[0] != 0 ||
+ ((const unsigned char *)loaded->gate.data)[0] != 7 ||
+ ((const unsigned char *)loaded->down.data)[0] != 23 ||
+ ((const unsigned char *)loaded->up.data)[0] != 39)
{ fprintf(stderr, "expert view mismatch: format=%d rows=%lld columns=%lld data=%zu scales=%zu bytes=%u/%u/%u/%u\n",
- (int)view.gate.format, (long long)view.gate.rows,
- (long long)view.gate.columns, view.gate.data_bytes,
- view.gate.scale_bytes,
- ((const unsigned char *)view.gate.scales)[0],
- ((const unsigned char *)view.gate.data)[0],
- ((const unsigned char *)view.down.data)[0],
- ((const unsigned char *)view.up.data)[0]); return 1; }
- coli_expert_release(store, &view);
+ (int)loaded->gate.format, (long long)loaded->gate.rows,
+ (long long)loaded->gate.columns, loaded->gate.data_bytes,
+ loaded->gate.scale_bytes,
+ ((const unsigned char *)loaded->gate.scales)[0],
+ ((const unsigned char *)loaded->gate.data)[0],
+ ((const unsigned char *)loaded->down.data)[0],
+ ((const unsigned char *)loaded->up.data)[0]); return 1; }
+ coli_expert_release(store, &same[0].view);
{
static const ColiExpertView zero;
- if (memcmp(&view, &zero, sizeof(view)) != 0) {
+ if (memcmp(&same[0].view, &zero, sizeof(same[0].view)) != 0) {
fprintf(stderr, "release did not clear view\n");
return 1;
}
}
/* Double release of a cleared view is a no-op. */
- coli_expert_release(store, &view);
- if (coli_expert_lookup(store, key, &view) != 0) return 1;
- coli_expert_release(store, &view);
+ coli_expert_release(store, &same[0].view);
+ coli_expert_release(store, &same[1].view);
/* Invalid key lookup must fail and clear the view. */
memset(&view, 0x3c, sizeof(view));
if (coli_expert_lookup(store, (ColiExpertKey){9, 9}, &view) == 0) return 1;
@@ -386,7 +518,7 @@ static int test_expert_store(void) {
store->ops->stats(store, &stats);
if (stats.requests != 2 || stats.hits != 1 || stats.misses != 1 ||
stats.prefetched != 1 || stats.bytes_read != 51 ||
- stats.resident_bytes != 51 || stats.capacity_bytes != 51)
+ stats.resident_bytes != 51 || stats.capacity_bytes != 153)
{ fprintf(stderr, "expert stats mismatch: requests=%llu hits=%llu misses=%llu prefetched=%llu bytes=%llu resident=%llu capacity=%llu\n",
(unsigned long long)stats.requests,
(unsigned long long)stats.hits,
@@ -395,6 +527,37 @@ static int test_expert_store(void) {
(unsigned long long)stats.bytes_read,
(unsigned long long)stats.resident_bytes,
(unsigned long long)stats.capacity_bytes); return 1; }
+
+ ExpertLookupJob distinct[2];
+ reset_expert_hooks(TEST_READ_DISTINCT_EXPERTS);
+ if (run_parallel_lookups(store, (ColiExpertKey){0, 1},
+ (ColiExpertKey){0, 2}, distinct) != 0) {
+ fprintf(stderr, "parallel distinct-expert lookup failed\n"); return 1;
+ }
+ if (expert_hook_reads != 2 || expert_hook_waits != 0 ||
+ expert_hook_max_active != 2) {
+ fprintf(stderr,
+ "distinct expert loads did not overlap: reads=%d waits=%d active=%d\n",
+ expert_hook_reads, expert_hook_waits,
+ expert_hook_max_active);
+ return 1;
+ }
+ coli_expert_release(store, &distinct[0].view);
+ coli_expert_release(store, &distinct[1].view);
+ store->ops->stats(store, &stats);
+ if (stats.requests != 4 || stats.hits != 1 || stats.misses != 3 ||
+ stats.prefetched != 1 || stats.bytes_read != 153 ||
+ stats.resident_bytes != 153 || stats.capacity_bytes != 153) {
+ fprintf(stderr,
+ "concurrent expert stats mismatch: requests=%llu hits=%llu misses=%llu bytes=%llu resident=%llu capacity=%llu\n",
+ (unsigned long long)stats.requests,
+ (unsigned long long)stats.hits,
+ (unsigned long long)stats.misses,
+ (unsigned long long)stats.bytes_read,
+ (unsigned long long)stats.resident_bytes,
+ (unsigned long long)stats.capacity_bytes);
+ return 1;
+ }
store->ops->destroy(store);
unlink(path);
rmdir(directory);