Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

improved c/deepseek_v4.c to allow the run of deepseek_v4_flash-0731-reap-150b (#1310)

Matteo Deliperi387653f54ecb
diff --git a/c/deepseek_v4.c b/c/deepseek_v4.c
index 944922ee..1cd2e08f 100644
--- a/c/deepseek_v4.c
+++ b/c/deepseek_v4.c
@@ -7255,12 +7255,23 @@ enum { V4_W1 = 0, V4_W2 = 1, V4_W3 = 2, V4_MATRIX_COUNT = 3 };
typedef struct {
const ColiSafetensorsTensor *weight[V4_MATRIX_COUNT];
const ColiSafetensorsTensor *scale[V4_MATRIX_COUNT];
- int shard;
+ int scale_shard;
+ int weight_shard;
uint64_t scale_offset;
uint64_t scale_bytes;
uint64_t weight_offset;
uint64_t weight_bytes;
uint64_t record_bytes;
+ /* Per-matrix fallback when scale (or weight) tensors are split across
+ * shards (REAP-style packed checkpoints). per_matrix=1 selects m_off/
+ * m_len/m_shard directly; the group fields above are ignored then. */
+ int per_matrix;
+ int m_scale_shard[V4_MATRIX_COUNT];
+ int m_weight_shard[V4_MATRIX_COUNT];
+ uint64_t m_scale_offset[V4_MATRIX_COUNT];
+ uint64_t m_scale_bytes[V4_MATRIX_COUNT];
+ uint64_t m_weight_offset[V4_MATRIX_COUNT];
+ uint64_t m_weight_bytes[V4_MATRIX_COUNT];
} V4ExpertRecord;
typedef struct {
@@ -7370,16 +7381,39 @@ static int build_record(V4ExpertStoreState *state, int layer, int expert,
layer, expert, matrix_names[matrix]);
}
int scale_shard = -1, weight_shard = -1;
- if (contiguous_group(record->scale, state->index,
- &scale_shard, &record->scale_offset,
- &record->scale_bytes) != 0 ||
- contiguous_group(record->weight, state->index,
- &weight_shard, &record->weight_offset,
- &record->weight_bytes) != 0 || scale_shard != weight_shard)
- return set_error(error, error_size,
- "expert is not two contiguous ranges: layer=%d expert=%d",
- layer, expert);
- record->shard = scale_shard;
+ int scale_range_contiguous = contiguous_group(record->scale, state->index,
+ &scale_shard, &record->scale_offset,
+ &record->scale_bytes) == 0;
+ int weight_range_contiguous = contiguous_group(record->weight, state->index,
+ &weight_shard, &record->weight_offset,
+ &record->weight_bytes) == 0;
+ if (!scale_range_contiguous || !weight_range_contiguous) {
+ /* REAP-style packed checkpoint: fall back to per-matrix reads. */
+ record->per_matrix = 1;
+ uint64_t per_matrix_bytes = 0;
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ int matrix_scale_shard = coli_st_tensor_shard(state->index, record->scale[matrix]);
+ int matrix_weight_shard = coli_st_tensor_shard(state->index, record->weight[matrix]);
+ if (matrix_scale_shard < 0 || matrix_weight_shard < 0)
+ return set_error(error, error_size,
+ "expert shard lookup failed: layer=%d expert=%d",
+ layer, expert);
+ record->m_scale_shard[matrix] = matrix_scale_shard;
+ record->m_weight_shard[matrix] = matrix_weight_shard;
+ record->m_scale_offset[matrix] = (uint64_t)record->scale[matrix]->off;
+ record->m_scale_bytes[matrix] = (uint64_t)record->scale[matrix]->nbytes;
+ record->m_weight_offset[matrix] = (uint64_t)record->weight[matrix]->off;
+ record->m_weight_bytes[matrix] = (uint64_t)record->weight[matrix]->nbytes;
+ per_matrix_bytes += record->m_scale_bytes[matrix] + record->m_weight_bytes[matrix];
+ }
+ record->scale_shard = record->m_scale_shard[0];
+ record->weight_shard = record->m_weight_shard[0];
+ record->record_bytes = per_matrix_bytes;
+ return 0;
+ }
+ record->per_matrix = 0;
+ record->scale_shard = scale_shard;
+ record->weight_shard = weight_shard;
record->record_bytes = record->scale_bytes + record->weight_bytes;
return 0;
}
@@ -7539,12 +7573,25 @@ static void fill_tensor_view(ColiTensorView *view,
const V4ExpertSlot *slot, int matrix) {
const ColiSafetensorsTensor *weight = record->weight[matrix];
const ColiSafetensorsTensor *scale = record->scale[matrix];
+ uint64_t scale_base = 0, weight_base = 0;
+ if (record->per_matrix) {
+ /* REAP fallback: slab packs per-matrix scales first, then weights. */
+ for (int prior = 0; prior < matrix; prior++)
+ scale_base += record->m_scale_bytes[prior];
+ for (int all = 0; all < V4_MATRIX_COUNT; all++)
+ weight_base += record->m_scale_bytes[all];
+ for (int prior = 0; prior < matrix; prior++)
+ weight_base += record->m_weight_bytes[prior];
+ } else {
+ scale_base = (uint64_t)scale->off - record->scale_offset;
+ weight_base = record->scale_bytes +
+ ((uint64_t)weight->off - record->weight_offset);
+ }
memset(view, 0, sizeof(*view));
view->format = COLI_TENSOR_FP4_NATIVE_BLOCK;
view->scale_format = COLI_SCALE_UE8M0;
- view->data = slot->slab + record->scale_bytes +
- ((uint64_t)weight->off - record->weight_offset);
- view->scales = slot->slab + ((uint64_t)scale->off - record->scale_offset);
+ view->data = slot->slab + weight_base;
+ view->scales = slot->slab + scale_base;
view->data_bytes = (size_t)weight->nbytes;
view->scale_bytes = (size_t)scale->nbytes;
view->rows = weight->shape[0];
@@ -7597,13 +7644,44 @@ static int lookup(ColiExpertStore *store, ColiExpertKey key,
int rep = coli_st_expert_route(key.layer, key.expert);
struct timespec disk_t0;
clock_gettime(CLOCK_MONOTONIC, &disk_t0);
- if (coli_st_read_at_streaming_rep(
- state->index, record->shard, rep, record->scale_offset,
- (size_t)record->scale_bytes, slot->slab) != 0 ||
- coli_st_read_at_streaming_rep(
- state->index, record->shard, rep, record->weight_offset,
- (size_t)record->weight_bytes,
- slot->slab + record->scale_bytes) != 0) {
+ int read_failed = 0;
+ if (record->per_matrix) {
+ uint64_t scale_cursor = 0;
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ if (coli_st_read_at_streaming_rep(
+ state->index, record->m_scale_shard[matrix], rep,
+ record->m_scale_offset[matrix],
+ (size_t)record->m_scale_bytes[matrix],
+ slot->slab + scale_cursor) != 0) {
+ read_failed = 1; break;
+ }
+ scale_cursor += record->m_scale_bytes[matrix];
+ }
+ if (!read_failed) {
+ uint64_t weight_cursor = 0;
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++)
+ weight_cursor += record->m_scale_bytes[matrix];
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT && !read_failed; matrix++) {
+ if (coli_st_read_at_streaming_rep(
+ state->index, record->m_weight_shard[matrix], rep,
+ record->m_weight_offset[matrix],
+ (size_t)record->m_weight_bytes[matrix],
+ slot->slab + weight_cursor) != 0)
+ read_failed = 1;
+ weight_cursor += record->m_weight_bytes[matrix];
+ }
+ }
+ } else {
+ if (coli_st_read_at_streaming_rep(
+ state->index, record->scale_shard, rep, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab) != 0 ||
+ coli_st_read_at_streaming_rep(
+ state->index, record->weight_shard, rep, record->weight_offset,
+ (size_t)record->weight_bytes,
+ slot->slab + record->scale_bytes) != 0)
+ read_failed = 1;
+ }
+ if (read_failed) {
struct timespec disk_t1;
clock_gettime(CLOCK_MONOTONIC, &disk_t1);
state->disk_sec +=
@@ -7666,7 +7744,7 @@ static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
V4ExpertStoreState *state = store->state;
int accepted = 0;
#ifdef COLI_V4_EXPERIMENTAL_PREFETCH_BATCH
- size_t capacity = count * 2, ranges = 0;
+ size_t capacity = count * 6, ranges = 0;
int *shards = malloc(capacity * sizeof(*shards));
uint64_t *offsets = malloc(capacity * sizeof(*offsets));
size_t *lengths = malloc(capacity * sizeof(*lengths));
@@ -7681,12 +7759,25 @@ static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
V4ExpertSlot *slot = indexed_expert_slot(state, keys[i]);
int resident = slot && slot->slab && slot->expert == keys[i].expert;
if (resident) continue;
- shards[ranges] = record->shard;
- offsets[ranges] = record->scale_offset;
- lengths[ranges++] = (size_t)record->scale_bytes;
- shards[ranges] = record->shard;
- offsets[ranges] = record->weight_offset;
- lengths[ranges++] = (size_t)record->weight_bytes;
+ if (record->per_matrix) {
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ shards[ranges] = record->m_scale_shard[matrix];
+ offsets[ranges] = record->m_scale_offset[matrix];
+ lengths[ranges++] = (size_t)record->m_scale_bytes[matrix];
+ }
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ shards[ranges] = record->m_weight_shard[matrix];
+ offsets[ranges] = record->m_weight_offset[matrix];
+ lengths[ranges++] = (size_t)record->m_weight_bytes[matrix];
+ }
+ } else {
+ shards[ranges] = record->scale_shard;
+ offsets[ranges] = record->scale_offset;
+ lengths[ranges++] = (size_t)record->scale_bytes;
+ shards[ranges] = record->weight_shard;
+ offsets[ranges] = record->weight_offset;
+ lengths[ranges++] = (size_t)record->weight_bytes;
+ }
candidates++;
}
pthread_mutex_unlock(&state->mutex);
@@ -7699,13 +7790,44 @@ static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
V4ExpertRecord *record = get_record(state, keys[i]);
if (!record) continue;
int rep = coli_st_expert_route(keys[i].layer, keys[i].expert);
- if (coli_st_prefetch_at_rep(state->index, record->shard, rep,
- record->scale_offset,
- (size_t)record->scale_bytes) == 0 &&
- coli_st_prefetch_at_rep(state->index, record->shard, rep,
- record->weight_offset,
- (size_t)record->weight_bytes) == 0)
- accepted++;
+ int all_weights_prefetched = 1;
+ int matrix_count = record->per_matrix ? V4_MATRIX_COUNT : 1;
+ for (int segment = 0; segment < matrix_count && all_weights_prefetched; segment++) {
+ int shard;
+ uint64_t offset;
+ size_t length;
+ if (record->per_matrix) {
+ shard = record->m_weight_shard[segment];
+ offset = record->m_weight_offset[segment];
+ length = (size_t)record->m_weight_bytes[segment];
+ } else {
+ shard = record->weight_shard;
+ offset = record->weight_offset;
+ length = (size_t)record->weight_bytes;
+ }
+ if (coli_st_prefetch_at_rep(state->index, shard, rep,
+ offset, length) != 0)
+ all_weights_prefetched = 0;
+ }
+ if (!all_weights_prefetched) continue;
+ int all_scales_prefetched = 1;
+ for (int segment = 0; segment < matrix_count; segment++) {
+ int shard;
+ uint64_t offset;
+ size_t length;
+ if (record->per_matrix) {
+ shard = record->m_scale_shard[segment];
+ offset = record->m_scale_offset[segment];
+ length = (size_t)record->m_scale_bytes[segment];
+ } else {
+ shard = record->scale_shard;
+ offset = record->scale_offset;
+ length = (size_t)record->scale_bytes;
+ }
+ all_scales_prefetched &= coli_st_prefetch_at_rep(state->index, shard, rep,
+ offset, length) == 0;
+ }
+ if (all_weights_prefetched && all_scales_prefetched) accepted++;
}
#endif
pthread_mutex_lock(&state->mutex);
@@ -8083,11 +8205,38 @@ static int v4_read_direct_window(const V4ExpertStoreState *state, int shard,
static int v4_read_expert_record(V4ExpertStoreState *state,
const V4ExpertRecord *record,
V4ExpertSlot *slot, int rep) {
+ if (record->per_matrix) {
+ int direct_available = slot->aligned_slab &&
+ coli_st_streaming_direct_available_rep(state->index, record->m_scale_shard[0], rep);
+ if (direct_available)
+ __atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),
+ __ATOMIC_RELAXED);
+ uint64_t scale_cursor = 0;
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ if (coli_st_read_at_rep(state->index, record->m_scale_shard[matrix], rep,
+ record->m_scale_offset[matrix],
+ (size_t)record->m_scale_bytes[matrix],
+ slot->slab + scale_cursor) != 0)
+ return -1;
+ scale_cursor += record->m_scale_bytes[matrix];
+ }
+ uint64_t weight_cursor = scale_cursor;
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ if (coli_st_read_at_rep(state->index, record->m_weight_shard[matrix], rep,
+ record->m_weight_offset[matrix],
+ (size_t)record->m_weight_bytes[matrix],
+ slot->slab + weight_cursor) != 0)
+ return -1;
+ weight_cursor += record->m_weight_bytes[matrix];
+ }
+ return 0;
+ }
int direct_available = slot->aligned_slab &&
- coli_st_streaming_direct_available_rep(state->index, record->shard, rep);
- if (direct_available &&
+ coli_st_streaming_direct_available_rep(state->index, record->scale_shard, rep);
+ int same_shard = record->scale_shard == record->weight_shard;
+ if (direct_available && same_shard &&
record->scale_offset + record->scale_bytes == record->weight_offset &&
- !v4_read_direct_window(state, record->shard, rep, slot->slab,
+ !v4_read_direct_window(state, record->scale_shard, rep, slot->slab,
record->scale_offset,
(size_t)record->record_bytes, 0)) {
__atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);
@@ -8099,24 +8248,24 @@ static int v4_read_expert_record(V4ExpertStoreState *state,
}
int weight_direct = direct_available && !v4_read_direct_window(
- state, record->shard, rep, slot->slab, record->weight_offset,
+ state, record->weight_shard, rep, slot->slab, record->weight_offset,
(size_t)record->weight_bytes, (size_t)record->scale_bytes);
if (weight_direct) {
__atomic_fetch_add(&v4_direct_reads, UINT64_C(1), __ATOMIC_RELAXED);
__atomic_fetch_add(&v4_direct_payload_bytes, record->weight_bytes,
__ATOMIC_RELAXED);
- return coli_st_read_at_rep(state->index, record->shard, rep,
+ return coli_st_read_at_rep(state->index, record->scale_shard, rep,
record->scale_offset,
(size_t)record->scale_bytes, slot->slab);
}
if (direct_available)
__atomic_fetch_add(&v4_direct_fallbacks, UINT64_C(1),
__ATOMIC_RELAXED);
- if (coli_st_read_at_rep(state->index, record->shard, rep,
+ if (coli_st_read_at_rep(state->index, record->weight_shard, rep,
record->weight_offset,
(size_t)record->weight_bytes,
slot->slab + record->scale_bytes)) return -1;
- return coli_st_read_at_rep(state->index, record->shard, rep,
+ return coli_st_read_at_rep(state->index, record->scale_shard, rep,
record->scale_offset,
(size_t)record->scale_bytes, slot->slab);
}
@@ -8649,7 +8798,7 @@ int COLI_V4_ROWS16_STORE_OPEN(
V4ExpertStoreState *state = (*output)->state;
int direct_io = state->layers > 0 && state->experts_per_layer > 0 &&
coli_st_streaming_direct_available(
- state->index, state->records[0].shard);
+ state->index, state->records[0].weight_shard);
fprintf(stderr, "v4_ssd_io mode=%s fallback=buffered-pread\n",
direct_io ? "direct-aligned" : "buffered-pread");
int minimum_slots = state->experts_per_layer < 6
@@ -14641,12 +14790,23 @@ enum { V4_W1 = 0, V4_W2 = 1, V4_W3 = 2, V4_MATRIX_COUNT = 3 };
typedef struct {
const ColiSafetensorsTensor *weight[V4_MATRIX_COUNT];
const ColiSafetensorsTensor *scale[V4_MATRIX_COUNT];
- int shard;
+ int scale_shard;
+ int weight_shard;
uint64_t scale_offset;
uint64_t scale_bytes;
uint64_t weight_offset;
uint64_t weight_bytes;
uint64_t record_bytes;
+ /* Per-matrix fallback when scale (or weight) tensors are split across
+ * shards (REAP-style packed checkpoints). per_matrix=1 selects m_off/
+ * m_len/m_shard directly; the group fields above are ignored then. */
+ int per_matrix;
+ int m_scale_shard[V4_MATRIX_COUNT];
+ int m_weight_shard[V4_MATRIX_COUNT];
+ uint64_t m_scale_offset[V4_MATRIX_COUNT];
+ uint64_t m_scale_bytes[V4_MATRIX_COUNT];
+ uint64_t m_weight_offset[V4_MATRIX_COUNT];
+ uint64_t m_weight_bytes[V4_MATRIX_COUNT];
} V4ExpertRecord;
typedef struct {
@@ -14736,16 +14896,39 @@ static int build_record(V4ExpertStoreState *state, int layer, int expert,
layer, expert, matrix_names[matrix]);
}
int scale_shard = -1, weight_shard = -1;
- if (contiguous_group(record->scale, state->index,
- &scale_shard, &record->scale_offset,
- &record->scale_bytes) != 0 ||
- contiguous_group(record->weight, state->index,
- &weight_shard, &record->weight_offset,
- &record->weight_bytes) != 0 || scale_shard != weight_shard)
- return set_error(error, error_size,
- "expert is not two contiguous ranges: layer=%d expert=%d",
- layer, expert);
- record->shard = scale_shard;
+ int scale_range_contiguous = contiguous_group(record->scale, state->index,
+ &scale_shard, &record->scale_offset,
+ &record->scale_bytes) == 0;
+ int weight_range_contiguous = contiguous_group(record->weight, state->index,
+ &weight_shard, &record->weight_offset,
+ &record->weight_bytes) == 0;
+ if (!scale_range_contiguous || !weight_range_contiguous) {
+ /* REAP-style packed checkpoint: fall back to per-matrix reads. */
+ record->per_matrix = 1;
+ uint64_t per_matrix_bytes = 0;
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ int matrix_scale_shard = coli_st_tensor_shard(state->index, record->scale[matrix]);
+ int matrix_weight_shard = coli_st_tensor_shard(state->index, record->weight[matrix]);
+ if (matrix_scale_shard < 0 || matrix_weight_shard < 0)
+ return set_error(error, error_size,
+ "expert shard lookup failed: layer=%d expert=%d",
+ layer, expert);
+ record->m_scale_shard[matrix] = matrix_scale_shard;
+ record->m_weight_shard[matrix] = matrix_weight_shard;
+ record->m_scale_offset[matrix] = (uint64_t)record->scale[matrix]->off;
+ record->m_scale_bytes[matrix] = (uint64_t)record->scale[matrix]->nbytes;
+ record->m_weight_offset[matrix] = (uint64_t)record->weight[matrix]->off;
+ record->m_weight_bytes[matrix] = (uint64_t)record->weight[matrix]->nbytes;
+ per_matrix_bytes += record->m_scale_bytes[matrix] + record->m_weight_bytes[matrix];
+ }
+ record->scale_shard = record->m_scale_shard[0];
+ record->weight_shard = record->m_weight_shard[0];
+ record->record_bytes = per_matrix_bytes;
+ return 0;
+ }
+ record->per_matrix = 0;
+ record->scale_shard = scale_shard;
+ record->weight_shard = weight_shard;
record->record_bytes = record->scale_bytes + record->weight_bytes;
return 0;
}
@@ -14766,12 +14949,25 @@ static void fill_tensor_view(ColiTensorView *view,
const V4ExpertSlot *slot, int matrix) {
const ColiSafetensorsTensor *weight = record->weight[matrix];
const ColiSafetensorsTensor *scale = record->scale[matrix];
+ uint64_t scale_base = 0, weight_base = 0;
+ if (record->per_matrix) {
+ /* REAP fallback: slab packs per-matrix scales first, then weights. */
+ for (int prior = 0; prior < matrix; prior++)
+ scale_base += record->m_scale_bytes[prior];
+ for (int all = 0; all < V4_MATRIX_COUNT; all++)
+ weight_base += record->m_scale_bytes[all];
+ for (int prior = 0; prior < matrix; prior++)
+ weight_base += record->m_weight_bytes[prior];
+ } else {
+ scale_base = (uint64_t)scale->off - record->scale_offset;
+ weight_base = record->scale_bytes +
+ ((uint64_t)weight->off - record->weight_offset);
+ }
memset(view, 0, sizeof(*view));
view->format = COLI_TENSOR_FP4_NATIVE_BLOCK;
view->scale_format = COLI_SCALE_UE8M0;
- view->data = slot->slab + record->scale_bytes +
- ((uint64_t)weight->off - record->weight_offset);
- view->scales = slot->slab + ((uint64_t)scale->off - record->scale_offset);
+ view->data = slot->slab + weight_base;
+ view->scales = slot->slab + scale_base;
view->data_bytes = (size_t)weight->nbytes;
view->scale_bytes = (size_t)scale->nbytes;
view->rows = weight->shape[0];
@@ -14829,13 +15025,44 @@ static int lookup(ColiExpertStore *store, ColiExpertKey key,
int rep = coli_st_expert_route(key.layer, key.expert);
struct timespec disk_t0;
clock_gettime(CLOCK_MONOTONIC, &disk_t0);
- if (coli_st_read_at_streaming_rep(
- state->index, record->shard, rep, record->scale_offset,
- (size_t)record->scale_bytes, slot->slab) != 0 ||
- coli_st_read_at_streaming_rep(
- state->index, record->shard, rep, record->weight_offset,
- (size_t)record->weight_bytes,
- slot->slab + record->scale_bytes) != 0) {
+ int read_failed = 0;
+ if (record->per_matrix) {
+ uint64_t scale_cursor = 0;
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ if (coli_st_read_at_streaming_rep(
+ state->index, record->m_scale_shard[matrix], rep,
+ record->m_scale_offset[matrix],
+ (size_t)record->m_scale_bytes[matrix],
+ slot->slab + scale_cursor) != 0) {
+ read_failed = 1; break;
+ }
+ scale_cursor += record->m_scale_bytes[matrix];
+ }
+ if (!read_failed) {
+ uint64_t weight_cursor = 0;
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++)
+ weight_cursor += record->m_scale_bytes[matrix];
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT && !read_failed; matrix++) {
+ if (coli_st_read_at_streaming_rep(
+ state->index, record->m_weight_shard[matrix], rep,
+ record->m_weight_offset[matrix],
+ (size_t)record->m_weight_bytes[matrix],
+ slot->slab + weight_cursor) != 0)
+ read_failed = 1;
+ weight_cursor += record->m_weight_bytes[matrix];
+ }
+ }
+ } else {
+ if (coli_st_read_at_streaming_rep(
+ state->index, record->scale_shard, rep, record->scale_offset,
+ (size_t)record->scale_bytes, slot->slab) != 0 ||
+ coli_st_read_at_streaming_rep(
+ state->index, record->weight_shard, rep, record->weight_offset,
+ (size_t)record->weight_bytes,
+ slot->slab + record->scale_bytes) != 0)
+ read_failed = 1;
+ }
+ if (read_failed) {
struct timespec disk_t1;
clock_gettime(CLOCK_MONOTONIC, &disk_t1);
state->disk_sec +=
@@ -14896,7 +15123,7 @@ static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
V4ExpertStoreState *state = store->state;
int accepted = 0;
#ifdef COLI_V4_EXPERIMENTAL_PREFETCH_BATCH
- size_t capacity = count * 2, ranges = 0;
+ size_t capacity = count * 6, ranges = 0;
int *shards = malloc(capacity * sizeof(*shards));
uint64_t *offsets = malloc(capacity * sizeof(*offsets));
size_t *lengths = malloc(capacity * sizeof(*lengths));
@@ -14915,12 +15142,25 @@ static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
resident = 1; break;
}
if (resident) continue;
- shards[ranges] = record->shard;
- offsets[ranges] = record->scale_offset;
- lengths[ranges++] = (size_t)record->scale_bytes;
- shards[ranges] = record->shard;
- offsets[ranges] = record->weight_offset;
- lengths[ranges++] = (size_t)record->weight_bytes;
+ if (record->per_matrix) {
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ shards[ranges] = record->m_scale_shard[matrix];
+ offsets[ranges] = record->m_scale_offset[matrix];
+ lengths[ranges++] = (size_t)record->m_scale_bytes[matrix];
+ }
+ for (int matrix = 0; matrix < V4_MATRIX_COUNT; matrix++) {
+ shards[ranges] = record->m_weight_shard[matrix];
+ offsets[ranges] = record->m_weight_offset[matrix];
+ lengths[ranges++] = (size_t)record->m_weight_bytes[matrix];
+ }
+ } else {
+ shards[ranges] = record->scale_shard;
+ offsets[ranges] = record->scale_offset;
+ lengths[ranges++] = (size_t)record->scale_bytes;
+ shards[ranges] = record->weight_shard;
+ offsets[ranges] = record->weight_offset;
+ lengths[ranges++] = (size_t)record->weight_bytes;
+ }
candidates++;
}
pthread_mutex_unlock(&state->mutex);
@@ -14933,13 +15173,44 @@ static int prefetch(ColiExpertStore *store, const ColiExpertKey *keys,
V4ExpertRecord *record = get_record(state, keys[i]);
if (!record) continue;
int rep = coli_st_expert_route(keys[i].layer, keys[i].expert);
- if (coli_st_prefetch_at_rep(state->index, record->shard, rep,
- record->scale_offset,
- (size_t)record->scale_bytes) == 0 &&
- coli_st_prefetch_at_rep(state->index, record->shard, rep,
- record->weight_offset,
- (size_t)record->weight_bytes) == 0)
- accepted++;
+ int all_weights_prefetched = 1;
+ int matrix_count = record->per_matrix ? V4_MATRIX_COUNT : 1;
+ for (int segment = 0; segment < matrix_count && all_weights_prefetched; segment++) {
+ int shard;
+ uint64_t offset;
+ size_t length;
+ if (record->per_matrix) {
+ shard = record->m_weight_shard[segment];
+ offset = record->m_weight_offset[segment];
+ length = (size_t)record->m_weight_bytes[segment];
+ } else {
+ shard = record->weight_shard;
+ offset = record->weight_offset;
+ length = (size_t)record->weight_bytes;
+ }
+ if (coli_st_prefetch_at_rep(state->index, shard, rep,
+ offset, length) != 0)
+ all_weights_prefetched = 0;
+ }
+ if (!all_weights_prefetched) continue;
+ int all_scales_prefetched = 1;
+ for (int segment = 0; segment < matrix_count; segment++) {
+ int shard;
+ uint64_t offset;
+ size_t length;
+ if (record->per_matrix) {
+ shard = record->m_scale_shard[segment];
+ offset = record->m_scale_offset[segment];
+ length = (size_t)record->m_scale_bytes[segment];
+ } else {
+ shard = record->scale_shard;
+ offset = record->scale_offset;
+ length = (size_t)record->scale_bytes;
+ }
+ all_scales_prefetched &= coli_st_prefetch_at_rep(state->index, shard, rep,
+ offset, length) == 0;
+ }
+ if (all_weights_prefetched && all_scales_prefetched) accepted++;
}
#endif
pthread_mutex_lock(&state->mutex);