Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
rans: harden the edges the review round reached β parse bounds, encoder bound, never-crash validator
monotophic4dc19b344788
diff --git a/c/.gitignore b/c/.gitignoreindex 3299f514..b77dca5a 100644--- a/c/.gitignore+++ b/c/.gitignore@@ -11,3 +11,5 @@ tools/iq3.dlltools/librans_c.sotools/librans_c.dylibtools/rans_c.dll+tests/fuzz_rans+tests/*.dSYM/diff --git a/c/Makefile b/c/Makefileindex b629d8ea..9247ab81 100644--- a/c/Makefile+++ b/c/Makefile@@ -460,6 +460,18 @@ tests/test_e8_kernel$(EXE): tests/test_e8_kernel.c quant.htests/test_rans$(EXE): tests/test_rans.c rans.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# Structured-mutation fuzz for the rans record parser/decoder under+# ASan+UBSan β seeded and bounded (a few thousand mutants, well under a+# minute), asserting no sanitizer report and byte identity across every+# compiled decode arm on every accepted record. NOT a test gate (sanitizer+# flags differ from the normal build); build+run on demand:+# make fuzz-rans+fuzz-rans: tests/fuzz_rans.c rans.h+ $(CC) -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all \+ -Wall -Wextra -Wno-unused-parameter -Wno-unused-function \+ tests/fuzz_rans.c -o tests/fuzz_rans -lm+ ./tests/fuzz_rans+tests/test_int3$(EXE): tests/test_int3.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)@@ -505,7 +517,12 @@ tests/test_pipe_block$(EXE): tests/test_pipe_block.c colibri.c st.h uring.h jsontest-c: $(TEST_BINS)$(PYTHON) tools/run_tests.py $(TEST_BINS)-test-python:+# test-python builds the rans ctypes bridge first: without it the repack e2e+# test's C-vs-Python byte-identity pin would silently compare Python against+# itself (the test ALSO skips loudly if the library is somehow still absent,+# e.g. when invoked via unittest directly β but a green `make test` must mean+# the pin was actually checked).+test-python: $(RANSLIB)$(PYTHON) -m unittest discover -s tests -p 'test_*.py'test: test-c test-python@@ -567,4 +584,4 @@ clean:bench: iobench$(EXE)@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) β run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi-.PHONY: all colibri glm iq3 rans cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench+.PHONY: all colibri glm iq3 rans fuzz-rans cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall benchdiff --git a/c/rans.h b/c/rans.hindex 0d355973..f0aaad81 100644--- a/c/rans.h+++ b/c/rans.h@@ -72,6 +72,15 @@#include <string.h>#include <stdint.h>+/* ENDIANNESS PRECONDITION: record header integers (n_symbols, packed_bytes,+ * stream_offsets) are little-endian on the wire, and this implementation+ * reads/writes them with native-order memcpy β correct only on little-endian+ * hosts. Refuse to compile elsewhere rather than emit byte-swapped records+ * that silently break the cross-host byte-identity contract. */+#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__)+#error "rans.h assumes a little-endian host (record integers are little-endian)"+#endif+#if defined(__AVX512F__) && defined(__AVX512BW__)#include <immintrin.h>#endif@@ -83,6 +92,7 @@#define RANS_ALPHABET 16#define RANS_NSTREAMS 256 /* int4-rans256-g0's stream count */#define RANS_SLACK 64 /* readable bytes required past record buffers */+#define RANS_SCALE_BITS_MAX 15 /* rans_table_init's upper bound on scale_bits *//* ---- scalar stream codec (ported unmodified from the proven prototype) --- */@@ -98,7 +108,11 @@ static size_t rans_encode_stream(const uint8_t *nibbles, size_t n,uint32_t x = RANS_L;uint8_t *ptr = out_buf + out_cap; /* write backwards from the end */for (size_t i = n; i-- > 0; ) {- uint32_t s = nibbles[i];+ /* mask to the nibble alphabet: an out-of-range input byte must never+ * index past freq[16]/start[16] (record-level entry points refuse+ * such input by name BEFORE encoding; the mask makes this primitive+ * memory-safe for any bytes regardless) */+ uint32_t s = nibbles[i] & 15u;uint32_t f = freq[s];uint32_t st = start[s];/* renormalize: keep x below x_max so the update keeps x in [L, L*256) */@@ -163,11 +177,14 @@ typedef enum {RANS_E_TRUNCATED, /* blob shorter than its own framing claims */RANS_E_EMPTY, /* n_symbols == 0 */RANS_E_COUNT_MISMATCH, /* packed_bytes != ceil(n_symbols/2) */+ RANS_E_OVERSIZE, /* n_symbols impossibly large for payload */RANS_E_OFFSET_FIRST, /* stream_offsets[0] != 0 */RANS_E_OFFSETS_MONOTONIC, /* stream_offsets not non-decreasing */RANS_E_STREAM_SHORT, /* a stream shorter than its 4-byte state */RANS_E_LENGTH_MISMATCH, /* blob length != derived framing length */RANS_E_PAD_NONZERO, /* a derived padding byte is not zero */+ RANS_E_UNALIGNED, /* record buffer not 4-byte aligned */+ RANS_E_NSTREAMS, /* n_streams zero / wrong for the format *//* table-level refusals (rans_table_init) */RANS_E_TABLE_SCALE, /* scale_bits out of range (1..15) */RANS_E_TABLE_FREQ_SUM, /* freq[] does not sum to M */@@ -178,6 +195,11 @@ typedef enum {RANS_E_STREAM_UNDERRUN, /* fewer than 4 bytes for the initial state */RANS_E_STREAM_LEFTOVER, /* decode finished with bytes unconsumed */RANS_E_STREAM_FINAL_STATE, /* final state != L (not an encoder output) */+ /* encode-side refusals */+ RANS_E_SYMBOL_UNCODABLE, /* input contains a value > 15 or a symbol+ whose table frequency is zero */+ RANS_E_SCRATCH, /* encoder scratch/output bound exhausted+ (a bound failure, NOT a memory condition) *//* misc */RANS_E_NOMEM,RANS_E_PATH_UNAVAILABLE /* RANS_PATH forced an arm this build/CPU/env@@ -190,11 +212,16 @@ static const char *rans_err_name(rans_err e) {case RANS_E_TRUNCATED: return "E_TRUNCATED";case RANS_E_EMPTY: return "E_EMPTY";case RANS_E_COUNT_MISMATCH: return "E_COUNT_MISMATCH";+ case RANS_E_OVERSIZE: return "E_OVERSIZE";case RANS_E_OFFSET_FIRST: return "E_OFFSET_FIRST";case RANS_E_OFFSETS_MONOTONIC: return "E_OFFSETS_MONOTONIC";case RANS_E_STREAM_SHORT: return "E_STREAM_SHORT";case RANS_E_LENGTH_MISMATCH: return "E_LENGTH_MISMATCH";case RANS_E_PAD_NONZERO: return "E_PAD_NONZERO";+ case RANS_E_UNALIGNED: return "E_UNALIGNED";+ case RANS_E_NSTREAMS: return "E_NSTREAMS";+ case RANS_E_SYMBOL_UNCODABLE: return "E_SYMBOL_UNCODABLE";+ case RANS_E_SCRATCH: return "E_SCRATCH";case RANS_E_TABLE_SCALE: return "E_TABLE_SCALE";case RANS_E_TABLE_FREQ_SUM: return "E_TABLE_FREQ_SUM";case RANS_E_TABLE_START: return "E_TABLE_START";@@ -306,38 +333,54 @@ static uint64_t rans_record_header_bytes(uint32_t n_streams) {return rans_round16(16u + ((uint64_t)n_streams + 1u) * 4u);}-/* Worst-case record size for n nibbles: per-stream worst case mirrors the- * encoder's own scratch rule (n + n/2 + 64 per stream is generous for any- * 16-symbol table whose present symbols have freq >= 1), plus framing. */+/* Worst-case record size for n nibbles. A symbol with table frequency f+ * costs at most log2(M/f) bits, so the per-stream worst case over any valid+ * table (f >= 1) is scale_bits bits/symbol β 1.875 bytes/symbol at the+ * maximum scale_bits of 15 β plus the 4 flushed state bytes. This bound+ * uses RANS_SCALE_BITS_MAX so it is table-independent and safe for every+ * table rans_table_init accepts (the old "n + n/2" rule was FALSE for+ * freq=1 symbols at scale_bits >= 13). Returns 0 for n_streams == 0. */static uint64_t rans_record_bound(uint64_t n_symbols, uint32_t n_streams) {+ if (n_streams == 0) return 0;uint64_t per = n_symbols / n_streams + 1u;+ uint64_t per_bytes = (per * RANS_SCALE_BITS_MAX + 7u) / 8u + 4u + 64u;return rans_record_header_bytes(n_streams) +- rans_round16((per + per / 2u + 64u) * n_streams);+ rans_round16(per_bytes * n_streams);}/* Encode n nibbles as one int4-rans256-g0 chunk record (n_streams round-robin* interleaved streams, shared table). Writes at most rans_record_bound()* bytes into out; *out_len receives the exact record length. Deterministic:- * same input + same table => byte-identical output. Returns RANS_OK, or- * RANS_E_EMPTY / RANS_E_NOMEM. */+ * same input + same table => byte-identical output. Returns RANS_OK, or:+ * RANS_E_EMPTY (n == 0), RANS_E_NSTREAMS (n_streams == 0),+ * RANS_E_SYMBOL_UNCODABLE (an input value > 15, or a symbol whose table+ * frequency is zero β detected by a pre-scan BEFORE any encoding work),+ * RANS_E_SCRATCH (a codec bound was exceeded β a logic/bound failure, never+ * reported as a memory condition), RANS_E_NOMEM (malloc failed). */static rans_err rans_record_encode(const uint8_t *nibbles, uint64_t n,const rans_table *t, uint32_t n_streams,uint8_t *out, uint64_t out_cap,uint64_t *out_len) {+ if (n_streams == 0) return RANS_E_NSTREAMS;if (n == 0) return RANS_E_EMPTY;+ /* pre-scan: refuse uncodable input by name, before touching `out` */+ for (uint64_t j = 0; j < n; j++)+ if (nibbles[j] > 15u || t->freq[nibbles[j]] == 0)+ return RANS_E_SYMBOL_UNCODABLE;uint64_t head = rans_record_header_bytes(n_streams);- if (out_cap < head) return RANS_E_NOMEM;+ if (out_cap < head) return RANS_E_SCRATCH;uint64_t sub_max = n / n_streams + 1u;- uint64_t scap = sub_max + sub_max / 2u + 64u; /* per-stream scratch cap */+ /* per-stream scratch: a freq>=1 symbol costs at most scale_bits bits, so+ * ceil(sub_max*scale_bits/8) + 4 flush bytes (+ slack) always fits */+ uint64_t scap = (sub_max * t->scale_bits + 7u) / 8u + 4u + 64u;uint8_t *sub = (uint8_t *)malloc(sub_max ? sub_max : 1);uint8_t *enc = (uint8_t *)malloc(scap);if (!sub || !enc) { free(sub); free(enc); return RANS_E_NOMEM; }memset(out, 0, head);- uint64_t packed_bytes = (n + 1u) / 2u;+ uint64_t packed_bytes = n / 2u + (n & 1u);memcpy(out, &n, 8);memcpy(out + 8, &packed_bytes, 8);- uint32_t *offs = (uint32_t *)(out + 16);uint64_t pos = 0; /* payload cursor */uint8_t *payload = out + head;@@ -346,17 +389,19 @@ static rans_err rans_record_encode(const uint8_t *nibbles, uint64_t n,for (uint64_t j = i; j < n; j += n_streams) sub[ns++] = nibbles[j];size_t off = rans_encode_stream(sub, ns, t->freq, t->start,t->scale_bits, enc, scap);- if (off == (size_t)-1) { free(sub); free(enc); return RANS_E_NOMEM; }+ if (off == (size_t)-1) { free(sub); free(enc); return RANS_E_SCRATCH; }uint64_t len = scap - off;- if (head + pos + len > out_cap) { free(sub); free(enc); return RANS_E_NOMEM; }+ if (head + pos + len > out_cap) { free(sub); free(enc); return RANS_E_SCRATCH; }memcpy(payload + pos, enc + off, len);- offs[i] = (uint32_t)pos;+ uint32_t pos32 = (uint32_t)pos;+ memcpy(out + 16 + (uint64_t)i * 4u, &pos32, 4);pos += len;- if (pos > 0xFFFFFFFFu) { free(sub); free(enc); return RANS_E_NOMEM; }+ if (pos > 0xFFFFFFFFu) { free(sub); free(enc); return RANS_E_SCRATCH; }}- offs[n_streams] = (uint32_t)pos;+ uint32_t total32 = (uint32_t)pos;+ memcpy(out + 16 + (uint64_t)n_streams * 4u, &total32, 4);uint64_t total = head + rans_round16(pos);- if (total > out_cap) { free(sub); free(enc); return RANS_E_NOMEM; }+ if (total > out_cap) { free(sub); free(enc); return RANS_E_SCRATCH; }memset(payload + pos, 0, rans_round16(pos) - pos);*out_len = total;free(sub); free(enc);@@ -376,30 +421,53 @@ typedef struct {/* Parse + validate one record blob (a tensor's full byte range). TRUST-* VERIFY-REFUSE: every malformation class returns its own named error and* the record is unusable on failure; there is no partial acceptance. The- * blob is borrowed. NOTE for the batched decode arms: the blob must satisfy- * the RANS_SLACK contract (header comment) β parsing alone does not read- * past blob_len. */+ * blob is borrowed and must be 4-byte aligned (any malloc'd or numpy-backed+ * buffer qualifies; refused by name otherwise) so the stream_offsets+ * pointer handed to the decode kernels indexes with defined behavior.+ * NOTE for the batched decode arms: the blob must additionally satisfy the+ * RANS_SLACK contract (header comment) β parsing alone does not read past+ * blob_len, and it performs no allocation (in particular nothing+ * proportional to the untrusted n_symbols field). */static rans_err rans_record_parse(const uint8_t *blob, uint64_t blob_len,uint32_t n_streams, rans_record *out) {memset(out, 0, sizeof(*out));+ if (n_streams == 0) return RANS_E_NSTREAMS;+ if (((uintptr_t)blob & 3u) != 0) return RANS_E_UNALIGNED;uint64_t head = rans_record_header_bytes(n_streams);if (blob_len < head) return RANS_E_TRUNCATED;uint64_t n_symbols, packed_bytes;memcpy(&n_symbols, blob, 8);memcpy(&packed_bytes, blob + 8, 8);if (n_symbols == 0) return RANS_E_EMPTY;- if (packed_bytes != (n_symbols + 1u) / 2u) return RANS_E_COUNT_MISMATCH;- const uint32_t *offs = (const uint32_t *)(blob + 16);- if (offs[0] != 0) return RANS_E_OFFSET_FIRST;+ /* non-wrapping form of packed_bytes == ceil(n_symbols/2): the additive+ * form (n_symbols+1)/2 wraps at UINT64_MAX and would accept+ * packed_bytes == 0 for it */+ if (packed_bytes != n_symbols / 2u + (n_symbols & 1u))+ return RANS_E_COUNT_MISMATCH;+ uint32_t off_prev, off_cur;+ memcpy(&off_prev, blob + 16, 4);+ if (off_prev != 0) return RANS_E_OFFSET_FIRST;for (uint32_t i = 0; i < n_streams; i++) {- if (offs[i + 1] < offs[i]) return RANS_E_OFFSETS_MONOTONIC;+ memcpy(&off_cur, blob + 16 + ((uint64_t)i + 1u) * 4u, 4);+ if (off_cur < off_prev) return RANS_E_OFFSETS_MONOTONIC;/* every stream carries at least its 4-byte flushed state (true even* for streams that encode zero symbols) */- if (offs[i + 1] - offs[i] < 4u) return RANS_E_STREAM_SHORT;+ if (off_cur - off_prev < 4u) return RANS_E_STREAM_SHORT;+ off_prev = off_cur;}- uint64_t payload_len = offs[n_streams];+ uint64_t payload_len = off_prev; /* == offsets[n_streams] */if (head + payload_len > blob_len) return RANS_E_TRUNCATED;if (blob_len != head + rans_round16(payload_len)) return RANS_E_LENGTH_MISMATCH;+ /* amplification bound: a symbol costs at least log2(M/(M-1)) bits under+ * any valid table (present symbols of a genuine record cost more; the+ * degenerate freq==M single-symbol table costs ~0 per symbol but still+ * satisfies this bound easily). n_symbols beyond payload_len*8*M_max is+ * impossible for ANY table this format admits β refuse the+ * decompression bomb here, before any consumer sizes buffers from+ * n_symbols. (No overflow: payload_len < 2^32, M_max = 2^15.) */+ if (n_symbols > payload_len * 8u * (uint64_t)(1u << RANS_SCALE_BITS_MAX))+ return RANS_E_OVERSIZE;+ const uint32_t *offs = (const uint32_t *)(blob + 16); /* aligned: checked *//* derived padding must be zero: bits hiding in the pad would make two* "identical" containers differ and defeat writer determinism */for (uint64_t i = 16u + ((uint64_t)n_streams + 1u) * 4u; i < head; i++)@@ -499,6 +567,40 @@ static int rans__selftest_arm(rans_path p) {rans_record_encode(data, NSYM, &t, NT, rec_buf, cap, &rec_len) == RANS_OK) {rans_record rec;if (rans_record_parse(rec_buf, rec_len, NT, &rec) == RANS_OK) {+ /* MACHINE-CHECKED band coverage: the load-bearing property of+ * this fixture is that it drives multi-byte renorm refills+ * (post-update x < 2^15 => refill count kb == 2) β the band a+ * comfortable table never visits and where a vector-arm renorm+ * bug hides. Count kb with the oracle-shaped loop and FAIL the+ * selftest if the fixture ever stops covering it, instead of+ * trusting a comment. (kb == 3 needs x < 2^7, impossible+ * mid-stream: post-update x >= L >> scale_bits >= 2^8.) */+ uint64_t kb2 = 0;+ for (uint32_t i = 0; i < NT; i++) {+ const uint8_t *ptr = rec.payload + rec.stream_offsets[i];+ const uint8_t *e2 = rec.payload + rec.stream_offsets[i + 1];+ uint32_t x = 0;+ for (int b2 = 0; b2 < 4; b2++) x = (x << 8) | *ptr++;+ for (uint64_t j = i; j < rec.n_symbols; j += NT) {+ uint32_t sl2 = x & (M - 1u);+ uint32_t sy2 = slot[sl2];+ x = freq[sy2] * (x >> sb) + sl2 - start[sy2];+ uint32_t kb = 0;+ while (x < RANS_L) {+ if (ptr >= e2) break;+ x = (x << 8) | *ptr++;+ kb++;+ }+ if (kb >= 2) kb2++;+ }+ }+ if (kb2 == 0) {+ fprintf(stderr, "[rans] selftest fixture lost multi-byte "+ "renorm coverage β fixture bug, arm not trusted\n");+ free(rec_buf);+ rans_table_free(&t);+ return 0;+ }memset(packed_ref, 0, sizeof(packed_ref));for (int i = 0; i < NSYM; i++)packed_ref[i >> 1] |= (uint8_t)(data[i] << ((i & 1) * 4));@@ -1015,6 +1117,9 @@ static void rans_record_decode_groups(const rans_record *rec, const rans_table *static rans_err rans_record_decode_packed(const rans_record *rec, const rans_table *t,uint32_t n_streams, rans_path p,uint8_t *out_packed) {+ /* n_streams == 0 would decode zero streams and return uninitialized+ * scratch as "successful" output β refuse before any path can run */+ if (n_streams == 0) return RANS_E_NSTREAMS;if (p == RANS_PATH_INVALID) return RANS_E_PATH_UNAVAILABLE;/* an arm this build did not compile refuses rather than silently running* scalar: a quiet downgrade would let "we tested <arm>" become a falsediff --git a/c/tests/fuzz_rans.c b/c/tests/fuzz_rans.cnew file mode 100644index 00000000..10daa303--- /dev/null+++ b/c/tests/fuzz_rans.c@@ -0,0 +1,177 @@+/* Structured-mutation fuzz for rans.h parse+decode against hostile records.+ * Every mutant blob is placed in a fresh heap buffer of exactly+ * blob_len + RANS_SLACK bytes so ASan catches any read past the documented+ * slack contract. Decode output buffer is exactly packed_bytes.+ *+ * Seeded and bounded (7 base records x 700 structured mutants). Build+run+ * via `make fuzz-rans` (ASan+UBSan, non-recoverable). Exit nonzero on any+ * cross-arm divergence; sanitizers abort on any memory/UB report. NOT part+ * of TEST_BINS: the sanitizer flags differ from the normal test build.+ * Adopted from the E1 review round's audit harness (its first run found the+ * u64-wrap acceptance in rans_record_parse that E_COUNT_MISMATCH's+ * non-wrapping form now refuses). */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <stdint.h>+#include "../rans.h"++static uint64_t rng_state = 0x9E3779B97F4A7C15ULL;+static uint32_t rnd(void) {+ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 7;+ rng_state ^= rng_state << 17; return (uint32_t)(rng_state >> 32);+}++static int parsed_ok = 0, refused = 0, decoded_ok = 0, decode_refused = 0,+ skipped_alloc = 0;++static void exercise(const uint8_t *mut, uint64_t mlen, const rans_table *t,+ uint32_t N) {+ /* fresh allocation: blob + slack ONLY */+ uint8_t *blob = (uint8_t *)malloc(mlen + RANS_SLACK);+ if (!blob) return;+ memcpy(blob, mut, mlen);+ memset(blob + mlen, 0, RANS_SLACK);+ rans_record rec;+ rans_err e = rans_record_parse(blob, mlen, N, &rec);+ if (e != RANS_OK) { refused++; free(blob); return; }+ parsed_ok++;+ if (rec.n_symbols > (1ull << 24) || rec.packed_bytes > (1ull << 28)) { skipped_alloc++; free(blob); return; }+ uint8_t *out = (uint8_t *)malloc(rec.packed_bytes ? rec.packed_bytes : 1);+ if (!out) { skipped_alloc++; free(blob); return; }+ const rans_path arms[] = {RANS_PATH_SCALAR, RANS_PATH_SCALAR_BF,+ RANS_PATH_NEON};+ uint8_t *ref = NULL;+ for (int a = 0; a < 3; a++) {+ memset(out, 0xEE, rec.packed_bytes);+ e = rans_record_decode_packed(&rec, t, N, arms[a], out);+ if (e != RANS_OK) { decode_refused++; continue; }+ decoded_ok++;+ if (a == 0) { ref = (uint8_t *)malloc(rec.packed_bytes);+ memcpy(ref, out, rec.packed_bytes); }+ else if (ref && memcmp(ref, out, rec.packed_bytes) != 0) {+ fprintf(stderr, "ARM DIVERGENCE on mutant (arm %d)\n", a);+ exit(2);+ }+ }+ /* checked per-stream decode over every stream too */+ for (uint32_t i = 0; i < N; i++) {+ uint64_t off0 = rec.stream_offsets[i], off1 = rec.stream_offsets[i + 1];+ uint64_t ns = 0;+ for (uint64_t j = i; j < rec.n_symbols; j += N) ns++;+ if (ns > (1ull << 26)) break;+ uint8_t *nib = (uint8_t *)malloc(ns ? ns : 1);+ rans_decode_stream_checked(rec.payload + off0, off1 - off0, ns,+ t->freq, t->start, t->slot_to_symbol,+ t->scale_bits, nib);+ free(nib);+ }+ free(ref); free(out); free(blob);+}++int main(void) {+ /* base table: the 15-symbol rare-mix shape */+ uint32_t sb = 14, M = 1u << 14;+ uint32_t freq[16] = {0}, start[16] = {0};+ for (int s = 1; s <= 7; s++) freq[s] = 1;+ for (int s = 8; s <= 14; s++) freq[s] = 3;+ freq[15] = M - 7 - 21;+ uint64_t acc = 0;+ for (int s = 0; s < 16; s++) { start[s] = (uint32_t)acc; acc += freq[s]; }+ uint16_t *slot = (uint16_t *)malloc(sizeof(uint16_t) * M);+ for (int s = 0; s < 16; s++)+ for (uint32_t i = 0; i < freq[s]; i++) slot[start[s] + i] = (uint16_t)s;+ rans_table t;+ if (rans_table_init(&t, sb, freq, start, slot) != RANS_OK) return 1;++ const uint32_t N = RANS_NSTREAMS;+ const uint64_t nsyms[] = {2, 255, 256, 511, 512, 8192, 65537};+ for (size_t bi = 0; bi < sizeof(nsyms) / sizeof(nsyms[0]); bi++) {+ uint64_t n = nsyms[bi];+ uint8_t *data = (uint8_t *)malloc(n);+ for (uint64_t i = 0; i < n; i++)+ data[i] = (uint8_t)((rnd() % 3 == 0) ? 1 + (rnd() % 14) : 15);+ uint64_t cap = rans_record_bound(n, N) + RANS_SLACK;+ uint8_t *base = (uint8_t *)calloc(cap, 1);+ uint64_t blen = 0;+ if (rans_record_encode(data, n, &t, N, base, cap, &blen) != RANS_OK)+ return 3;+ /* sanity: unmutated parses+decodes */+ exercise(base, blen, &t, N);++ uint8_t *mut = (uint8_t *)malloc(blen + 64);+ for (int it = 0; it < 700; it++) {+ uint64_t mlen = blen;+ memcpy(mut, base, blen);+ switch (it % 7) {+ case 0: { /* random byte corruption, 1-8 bytes anywhere */+ int k = 1 + (int)(rnd() % 8);+ for (int j = 0; j < k; j++) mut[rnd() % blen] ^= (uint8_t)(1 + (rnd() & 0xFF));+ break; }+ case 1: { /* header field attacks */+ uint64_t v;+ switch (rnd() % 6) {+ case 0: v = 0; break;+ case 1: v = UINT64_MAX; break;+ case 2: v = UINT64_MAX - 1; break;+ case 3: v = (uint64_t)1 << 32; break;+ case 4: v = n + 1; break;+ default: v = n - 1; break;+ }+ if (rnd() & 1) memcpy(mut, &v, 8);+ else memcpy(mut + 8, &v, 8);+ /* half the time keep packed_bytes consistent with lying n */+ if (rnd() & 1) {+ uint64_t nn; memcpy(&nn, mut, 8);+ uint64_t pb = (nn + 1) / 2;+ memcpy(mut + 8, &pb, 8);+ }+ break; }+ case 2: { /* offsets attacks */+ uint32_t idx = rnd() % (N + 1);+ uint32_t v;+ switch (rnd() % 5) {+ case 0: v = 0xFFFFFFFFu; break;+ case 1: v = 0; break;+ case 2: v = (uint32_t)(rnd()); break;+ case 3: { uint32_t cur; memcpy(&cur, mut + 16 + 4 * (uint64_t)idx, 4);+ v = cur + 1; break; }+ default: { uint32_t cur; memcpy(&cur, mut + 16 + 4 * (uint64_t)idx, 4);+ v = cur ? cur - 1 : 0; break; }+ }+ memcpy(mut + 16 + 4 * (uint64_t)idx, &v, 4);+ break; }+ case 3: /* truncate */+ mlen = rnd() % blen;+ break;+ case 4: /* extend with garbage */+ mlen = blen + 1 + rnd() % 63;+ for (uint64_t j = blen; j < mlen; j++) mut[j] = (uint8_t)rnd();+ break;+ case 5: { /* swap two offsets (reversed ranges) */+ uint32_t i1 = rnd() % (N + 1), i2 = rnd() % (N + 1);+ uint32_t a1, a2;+ memcpy(&a1, mut + 16 + 4 * (uint64_t)i1, 4);+ memcpy(&a2, mut + 16 + 4 * (uint64_t)i2, 4);+ memcpy(mut + 16 + 4 * (uint64_t)i1, &a2, 4);+ memcpy(mut + 16 + 4 * (uint64_t)i2, &a1, 4);+ break; }+ default: { /* payload bit flips near stream boundaries */+ uint32_t i1 = rnd() % N;+ uint32_t o; memcpy(&o, base + 16 + 4 * (uint64_t)i1, 4);+ uint64_t head = rans_record_header_bytes(N);+ uint64_t p = head + o + (rnd() % 8);+ if (p < blen) mut[p] ^= (uint8_t)(1u << (rnd() % 8));+ break; }+ }+ if (mlen == 0) { refused++; continue; }+ exercise(mut, mlen, &t, N);+ }+ free(mut); free(base); free(data);+ }+ printf("fuzz done: parsed_ok=%d refused=%d decoded_ok=%d decode_refused=%d skipped=%d\n",+ parsed_ok, refused, decoded_ok, decode_refused, skipped_alloc);+ rans_table_free(&t);+ free(slot);+ return 0;+}diff --git a/c/tests/test_rans.c b/c/tests/test_rans.cindex 11dcd81b..665865e6 100644--- a/c/tests/test_rans.c+++ b/c/tests/test_rans.c@@ -615,6 +615,233 @@ static void test_path_envelope(void) {#undef UNSET_ENV}+/* ---- 5. fix-round hardening regressions ---------------------------------- */++/* Header-field overflow/amplification battery (u64 wrap + decompression+ * bomb): reproduces the review round's poc_overflow.c cases and pins the+ * non-wrapping count check + the E_OVERSIZE payload bound. */+static void test_header_field_battery(void) {+ const uint32_t N = RANS_NSTREAMS;+ const uint64_t n = 8192;+ uint8_t *d = (uint8_t *)malloc(n);+ for (uint64_t i = 0; i < n; i++) d[i] = (uint8_t)(1 + (rnd() % 15));+ test_table tt;+ tt_from_hist(&tt, 14, d, n);+ uint64_t rec_len = 0;+ uint8_t *buf = make_record(d, n, &tt, N, &rec_len);+ uint8_t *cp = (uint8_t *)malloc(rec_len + RANS_SLACK);+ rans_record rec;+ rans_err e;++ struct { uint64_t n_symbols, packed_bytes; rans_err want; const char *why; }+ cases[] = {+ {UINT64_MAX, 0, RANS_E_COUNT_MISMATCH,+ "u64 wrap: (n+1)/2 overflows to 0"},+ {UINT64_MAX, (1ull << 63), RANS_E_OVERSIZE,+ "u64 max with non-wrapping-consistent pb"},+ {UINT64_MAX - 1, (1ull << 63) - 1, RANS_E_OVERSIZE, "u64 max-1"},+ {1ull << 63, 1ull << 62, RANS_E_OVERSIZE, "2^63"},+ {1ull << 40, 1ull << 39, RANS_E_OVERSIZE, "decompression bomb 2^40"},+ {1ull << 32, 1ull << 31, RANS_E_OVERSIZE, "2^32"},+ {(1ull << 32) + 1, (1ull << 31) + 1, RANS_E_OVERSIZE, "2^32+1"},+ {0, 0, RANS_E_EMPTY, "zero symbols"},+ {n + 1, (n + 1) / 2, RANS_E_COUNT_MISMATCH, "n+1, stale pb"},+ };+ for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) {+ memcpy(cp, buf, rec_len);+ memcpy(cp, &cases[c].n_symbols, 8);+ memcpy(cp + 8, &cases[c].packed_bytes, 8);+ e = rans_record_parse(cp, rec_len, N, &rec);+ CHECK(e == cases[c].want, "%s: want %s got %s", cases[c].why,+ rans_err_name(cases[c].want), rans_err_name(e));+ }++ /* minimally-framed bomb (every stream exactly its 4 flushed bytes,+ * n_symbols = 2^40): must refuse at parse, which allocates nothing */+ {+ uint64_t head = rans_record_header_bytes(N);+ uint64_t payload_len = 4ull * N;+ uint64_t blen = head + rans_round16(payload_len);+ uint8_t *bomb = (uint8_t *)calloc(blen + RANS_SLACK, 1);+ uint64_t nb = 1ull << 40, pb = nb / 2;+ memcpy(bomb, &nb, 8);+ memcpy(bomb + 8, &pb, 8);+ for (uint32_t i = 0; i <= N; i++) {+ uint32_t v = 4u * i;+ memcpy(bomb + 16 + (uint64_t)i * 4u, &v, 4);+ }+ for (uint64_t i = head; i < head + payload_len; i++) bomb[i] = 0x80;+ e = rans_record_parse(bomb, blen, N, &rec);+ CHECK(e == RANS_E_OVERSIZE, "minimal bomb: want E_OVERSIZE got %s",+ rans_err_name(e));+ free(bomb);+ }+ free(cp); free(buf); free(d);+ tt_free(&tt);+}++/* Encoder scratch bound: bulk freq=1 input must encode fine at every+ * scale_bits (the old n+n/2 rule failed at >= 13 β review poc_bound.c). */+static void test_encoder_scratch_bound(void) {+ for (uint32_t sb = 12; sb <= 15; sb++) {+ uint32_t M = 1u << sb;+ uint32_t freq[16] = {0};+ freq[3] = 1;+ freq[15] = M - 1;+ test_table tt;+ tt_build(&tt, sb, freq);+ const uint64_t n = 8192;+ uint8_t *d = (uint8_t *)malloc(n);+ memset(d, 3, n); /* every symbol is the rare one */+ uint64_t cap = rans_record_bound(n, 16) + RANS_SLACK, rl = 0;+ uint8_t *rb = (uint8_t *)calloc(cap, 1);+ rans_err e = rans_record_encode(d, n, &tt.t, 16, rb, cap, &rl);+ CHECK(e == RANS_OK, "freq=1 bulk encode at sb=%u: %s", sb,+ rans_err_name(e));+ if (e == RANS_OK) { /* and it round-trips */+ rans_record rec;+ CHECK(rans_record_parse(rb, rl, 16, &rec) == RANS_OK,+ "freq=1 sb=%u parse", sb);+ uint8_t *out = (uint8_t *)malloc(n / 2);+ uint8_t *ref = (uint8_t *)malloc(n / 2);+ pack_ref(d, n, ref);+ CHECK(rans_record_decode_packed(&rec, &tt.t, 16, RANS_PATH_SCALAR,+ out) == RANS_OK &&+ memcmp(out, ref, n / 2) == 0, "freq=1 sb=%u round trip", sb);+ free(out); free(ref);+ }+ /* E_SCRATCH is a bound failure, never conflated with malloc failure:+ * a too-small caller buffer names the bound, not the allocator */+ e = rans_record_encode(d, n, &tt.t, 16, rb,+ rans_record_header_bytes(16) + 8, &rl);+ CHECK(e == RANS_E_SCRATCH, "small out_cap at sb=%u: want E_SCRATCH "+ "got %s", sb, rans_err_name(e));+ free(rb); free(d);+ tt_free(&tt);+ }+}++/* Uncodable input is refused by name BEFORE any encoding work: a present+ * symbol with table freq 0, and out-of-alphabet bytes (review poc_encode.c).+ * The stream primitive itself masks to the nibble alphabet, so garbage+ * bytes can never index past freq[16] (byte-identical to pre-masked input). */+static void test_uncodable_input(void) {+ uint32_t freq[16] = {0};+ freq[1] = 1; freq[2] = 1; freq[3] = 1;+ freq[15] = (1u << 14) - 3; /* 4..14 have freq 0 */+ test_table tt;+ tt_build(&tt, 14, freq);+ uint8_t d[64];+ memset(d, 15, sizeof(d));+ uint64_t cap = rans_record_bound(64, 16) + RANS_SLACK, rl = 0;+ uint8_t *rb = (uint8_t *)calloc(cap, 1);+ rans_err e;++ d[7] = 9; /* freq[9] == 0 */+ e = rans_record_encode(d, 64, &tt.t, 16, rb, cap, &rl);+ CHECK(e == RANS_E_SYMBOL_UNCODABLE, "freq=0 symbol: want "+ "E_SYMBOL_UNCODABLE got %s", rans_err_name(e));++ d[7] = 200; /* out of the nibble alphabet */+ e = rans_record_encode(d, 64, &tt.t, 16, rb, cap, &rl);+ CHECK(e == RANS_E_SYMBOL_UNCODABLE, "value 200: want E_SYMBOL_UNCODABLE "+ "got %s", rans_err_name(e));++ /* stream primitive: masked, so 200 encodes as 200&15 == 8, byte-exact */+ uint32_t f2[16];+ for (int s = 0; s < 16; s++) f2[s] = (1u << 14) / 16;+ test_table t2;+ tt_build(&t2, 14, f2);+ uint8_t da[64], db[64], ba[512], bb[512];+ for (int i = 0; i < 64; i++) da[i] = db[i] = (uint8_t)(rnd() & 15);+ da[3] = 200; db[3] = 200 & 15;+ size_t oa = rans_encode_stream(da, 64, t2.t.freq, t2.t.start,+ t2.t.scale_bits, ba, sizeof(ba));+ size_t ob = rans_encode_stream(db, 64, t2.t.freq, t2.t.start,+ t2.t.scale_bits, bb, sizeof(bb));+ CHECK(oa == ob && oa != (size_t)-1 &&+ memcmp(ba + oa, bb + ob, sizeof(ba) - oa) == 0,+ "stream mask: 200 must encode as 200&15, byte-identical");+ free(rb);+ tt_free(&tt); tt_free(&t2);+}++/* n_streams validation + record-buffer alignment refusals. */+static void test_nstreams_and_alignment(void) {+ const uint64_t n = 512;+ uint8_t *d = (uint8_t *)malloc(n);+ for (uint64_t i = 0; i < n; i++) d[i] = (uint8_t)(1 + (rnd() % 15));+ test_table tt;+ tt_from_hist(&tt, 14, d, n);+ uint64_t rec_len = 0;+ uint8_t *buf = make_record(d, n, &tt, RANS_NSTREAMS, &rec_len);+ rans_record rec;+ rans_err e;++ CHECK(rans_record_bound(n, 0) == 0, "bound with n_streams=0 returns 0");+ uint64_t rl = 0;+ e = rans_record_encode(d, n, &tt.t, 0, buf, rec_len, &rl);+ CHECK(e == RANS_E_NSTREAMS, "encode n_streams=0: %s", rans_err_name(e));+ e = rans_record_parse(buf, rec_len, 0, &rec);+ CHECK(e == RANS_E_NSTREAMS, "parse n_streams=0: %s", rans_err_name(e));+ CHECK(rans_record_parse(buf, rec_len, RANS_NSTREAMS, &rec) == RANS_OK,+ "control parse");+ uint8_t out[256];+ e = rans_record_decode_packed(&rec, &tt.t, 0, RANS_PATH_SCALAR, out);+ CHECK(e == RANS_E_NSTREAMS, "decode n_streams=0: %s", rans_err_name(e));++ /* a misaligned record buffer is refused by name, not cast through */+ uint8_t *mis = (uint8_t *)malloc(rec_len + RANS_SLACK + 1);+ memcpy(mis + 1, buf, rec_len);+ e = rans_record_parse(mis + 1, rec_len, RANS_NSTREAMS, &rec);+ CHECK(e == RANS_E_UNALIGNED, "misaligned blob: %s", rans_err_name(e));+ free(mis); free(buf); free(d);+ tt_free(&tt);+}++/* Band coverage as a checked property of the suite's own rare-symbol+ * fixture (the runtime selftest asserts the same on its fixture): the+ * multi-byte renorm band the sabotage exercise proved load-bearing must+ * actually be visited, or the identity sweep is running blind. */+static void test_band_coverage(void) {+ const uint64_t n = 32768;+ uint8_t *d = (uint8_t *)malloc(n);+ uint32_t rare[16] = {0};+ for (int s = 1; s <= 7; s++) rare[s] = 1;+ for (int s = 8; s <= 14; s++) rare[s] = 3;+ rare[15] = (1u << 14) - 7 - 21;+ test_table tt;+ tt_build(&tt, 14, rare);+ for (uint64_t i = 0; i < n; i++)+ d[i] = (uint8_t)((rnd() % 3 == 0) ? 1 + (rnd() % 14) : 15);+ uint64_t rec_len = 0;+ uint8_t *buf = make_record(d, n, &tt, RANS_NSTREAMS, &rec_len);+ rans_record rec;+ CHECK(rans_record_parse(buf, rec_len, RANS_NSTREAMS, &rec) == RANS_OK,+ "band fixture parse");+ uint64_t kb2 = 0, steps = 0;+ for (uint32_t i = 0; i < RANS_NSTREAMS; i++) {+ const uint8_t *ptr = rec.payload + rec.stream_offsets[i];+ const uint8_t *end = rec.payload + rec.stream_offsets[i + 1];+ uint32_t x = 0;+ for (int b = 0; b < 4; b++) x = (x << 8) | *ptr++;+ for (uint64_t j = i; j < rec.n_symbols; j += RANS_NSTREAMS) {+ uint32_t slot = x & (tt.t.M - 1u);+ uint32_t s = tt.slot[slot];+ x = tt.t.freq[s] * (x >> tt.t.scale_bits) + slot - tt.t.start[s];+ uint32_t kb = 0;+ while (x < RANS_L) { if (ptr >= end) break; x = (x << 8) | *ptr++; kb++; }+ if (kb >= 2) kb2++;+ steps++;+ }+ }+ CHECK(kb2 > steps / 100, "multi-byte renorm band: %llu of %llu steps "+ "(need > 1%%) β the rare-symbol fixture lost its coverage",+ (unsigned long long)kb2, (unsigned long long)steps);+ free(buf); free(d);+ tt_free(&tt);+}+int main(void) {test_stream_property_suite();test_stream_checked_refusals();@@ -622,6 +849,11 @@ int main(void) {test_record_refusals();test_arm_identity_suite();test_path_envelope();+ test_header_field_battery();+ test_encoder_scratch_bound();+ test_uncodable_input();+ test_nstreams_and_alignment();+ test_band_coverage();if (failures) {printf("test_rans: FAIL (%d)\n", failures);return 1;diff --git a/c/tests/test_rans_repack.py b/c/tests/test_rans_repack.pyindex 9ea8a268..544e79c5 100644--- a/c/tests/test_rans_repack.py+++ b/c/tests/test_rans_repack.py@@ -44,7 +44,8 @@ def run_tool(script, *args, env_extra=None):@unittest.skipIf(rf is None, "numpy not available (offline-tooling test)")class TestRansRepack(unittest.TestCase):- O, I = 8, 128 # per-row ratio I/2 = 64 (safely above the g64 signature)+ O, I = 8, 256 # per-row ratio I/2 = 128, above every grouped+ # signature (32 = g64, 64 = g128/ambiguous)def setUp(self):self.tmp = tempfile.TemporaryDirectory()@@ -107,15 +108,32 @@ class TestRansRepack(unittest.TestCase):name + ".qs")self.assertEqual(qblob, qs, name + ".qs")- # determinism: an independent second run is byte-identical+ # determinism: an independent second run is byte-identical, manifest+ # (the completion marker) includedout2 = self.repack(self.root / "out2")self.assertEqual(out1[0].read_bytes(), out2[0].read_bytes())+ self.assertEqual(+ (out1[0].parent / "repack-manifest.json").read_bytes(),+ (out2[0].parent / "repack-manifest.json").read_bytes())# implementation identity: a forced pure-Python run emits the SAME- # bytes as the (lib-backed, when built) default run+ # bytes as the C-bridge run. VACUOUS unless the bridge is actually+ # loaded, so that is asserted β `make test-python` builds the bridge;+ # a direct unittest invocation without it skips LOUDLY instead of+ # green-lighting a pin it never checked.+ if rf.LIB is None:+ self.skipTest("tools/librans_c not built (run `make rans`): "+ "the C-vs-Python byte-identity pin was NOT checked")out3 = self.repack(self.root / "out3", RANS_FORCE_PYTHON="1")self.assertEqual(out1[0].read_bytes(), out3[0].read_bytes())+ # bridge-presence: the default run must actually have used the C+ # codec (not fallen back to Python with the lib sitting there)+ r = run_tool("repack_rans.py", "--indir", str(self.indir),+ "--outdir", str(self.root / "outdry"), "--dry-run")+ self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr)+ self.assertIn("codec: C (librans_c)", r.stdout)+# the validator trusts itr = run_tool("rans_verify.py", str(out1[0]))self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr)@@ -137,9 +155,12 @@ class TestRansRepack(unittest.TestCase):meta = meta_mutator(meta)rf.write_shard(str(shard), tensors, meta)+ _corrupt_seq = 0+def _corrupt_reshard_and_expect(self, expected_code, meta_mutator=None,tensor_mutator=None):- outdir = self.root / f"corrupt_{expected_code}"+ TestRansRepack._corrupt_seq += 1+ outdir = self.root / f"corrupt_{TestRansRepack._corrupt_seq}_{expected_code}"shard = self.repack(outdir)[0]self._reshard(shard, meta_mutator, tensor_mutator)r = run_tool("rans_verify.py", str(shard))@@ -177,6 +198,15 @@ class TestRansRepack(unittest.TestCase):blob[16 + (rf.N_STREAMS + 1) * 4] = 0x77 # header pad bytereturn blob+ def bomb_header(name, blob):+ if name == target:+ # n_symbols = 2^40 with a consistent packed_bytes: the+ # "oversize fields" class the spec names β must refuse as+ # E_OVERSIZE, never an allocator traceback+ blob[0:8] = (1 << 40).to_bytes(8, "little")+ blob[8:16] = (1 << 39).to_bytes(8, "little")+ return blob+def drop_stamp(meta):del meta[rf.METADATA_KEY]return meta@@ -206,6 +236,7 @@ class TestRansRepack(unittest.TestCase):# payload corruption trips a stream invariant or the re-encode pin;# both are refusals β assert on the shared prefixself._corrupt_reshard_and_expect("E_", tensor_mutator=flip_payload)+ self._corrupt_reshard_and_expect("E_OVERSIZE", tensor_mutator=bomb_header)self._corrupt_reshard_and_expect("E_COUNT_MISMATCH",tensor_mutator=flip_count)self._corrupt_reshard_and_expect("E_", tensor_mutator=break_offsets)@@ -235,6 +266,46 @@ class TestRansRepack(unittest.TestCase):self.assertEqual(r.returncode, 1)self.assertIn("E_GEOMETRY_G64", r.stderr)+ # gs=128 signature (upstream grouped-int4's actual group size):+ # ratio exactly 64, byte-indistinguishable from per-row I=128 ->+ # its own named refusal, never silently accepted as per-row+ g128dir = self.root / "g128"+ g128dir.mkdir()+ rows128 = wb // 64+ rf.write_shard(str(g128dir / "out-00000.safetensors"), [+ ("model.layers.0.mlp.experts.0.gate_proj.weight", "U8", [wb],+ bytes(wb)),+ ("model.layers.0.mlp.experts.0.gate_proj.weight.qs", "F32",+ [rows128], bytes(rows128 * 4)),+ ], {})+ r = run_tool("repack_rans.py", "--indir", str(g128dir),+ "--outdir", str(self.root / "g128out"))+ self.assertEqual(r.returncode, 1)+ self.assertIn("E_GEOMETRY_AMBIGUOUS", r.stderr)++ # 2-D weight shape: per-row geometry is verified POSITIVELY from the+ # shape fields β accepted when qs rows == O, refused by name when not+ rng = np.random.default_rng(3)+ O, rb2 = 8, 64+ nib = rng.choice(np.arange(1, 16, dtype=np.uint8), size=O * rb2 * 2)+ raw = rf.pack_nibbles(nib)+ for rows2d, expect_ok in ((O, True), (O * 2, False)):+ d2 = self.root / f"twod_{rows2d}"+ d2.mkdir()+ rf.write_shard(str(d2 / "out-00000.safetensors"), [+ ("model.layers.0.mlp.experts.0.gate_proj.weight", "U8",+ [O, rb2], raw),+ ("model.layers.0.mlp.experts.0.gate_proj.weight.qs", "F32",+ [rows2d], bytes(rows2d * 4)),+ ], {})+ r = run_tool("repack_rans.py", "--indir", str(d2),+ "--outdir", str(self.root / f"twod_out_{rows2d}"))+ if expect_ok:+ self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr)+ else:+ self.assertEqual(r.returncode, 1)+ self.assertIn("E_GEOMETRY_NOT_PER_ROW", r.stderr)+# missing .qs sidecar -> named refusalnsdir = self.root / "noscales"nsdir.mkdir()@@ -258,6 +329,162 @@ class TestRansRepack(unittest.TestCase):self.assertEqual(r.returncode, 1)self.assertIn("refusing to emit an empty container", r.stderr)+ def test_parity_c_python_refusal_classes(self):+ """Same bytes into the C parser (via the bridge) and the Python+ parser must produce the same named class β parity is a tested+ property, not a documented intention. Covers the u64-wrap and+ decompression-bomb header battery."""+ if rf.LIB is None:+ self.skipTest("tools/librans_c not built (run `make rans`): "+ "C/Python refusal parity was NOT checked")+ import ctypes++ def c_parse(blob):+ buf = np.zeros(len(blob) + rf.SLACK, dtype=np.uint8)+ buf[:len(blob)] = np.frombuffer(blob, dtype=np.uint8)+ ns = ctypes.c_uint64()+ pb = ctypes.c_uint64()+ rc = rf.LIB.rc_record_parse(rf._p8(buf), len(blob), rf.N_STREAMS,+ ctypes.byref(ns), ctypes.byref(pb))+ return "OK" if rc == 0 else rf.LIB.rc_err_name(rc).decode()++ def py_parse(blob):+ try:+ rf.parse_record(blob)+ return "OK"+ except rf.RansRefusal as exc:+ return exc.code++ rng = np.random.default_rng(11)+ n = 4096+ nib = rng.choice(np.arange(1, 16, dtype=np.uint8), size=n)+ hist = np.bincount(nib, minlength=16)+ freq = rf.quantize_freq(hist)+ start, slot = rf.build_table(freq)+ base = bytearray(rf.build_record(nib, freq, start, slot))+ u64max = (1 << 64) - 1++ def hdr(ns, pb):+ b = bytearray(base)+ b[0:8] = ns.to_bytes(8, "little")+ b[8:16] = pb.to_bytes(8, "little")+ return bytes(b)++ cases = [bytes(base)] # the valid control+ for ns, pb in ((u64max, 0), (u64max, 1 << 63), (0, 0),+ (1 << 40, 1 << 39), (1 << 32, 1 << 31),+ (n + 1, (n + 1) // 2), (n, n)):+ cases.append(hdr(ns, pb))+ cases.append(bytes(base[:64])) # truncated header+ cases.append(bytes(base[:len(base) - 16])) # truncated payload+ cases.append(bytes(base) + b"\x00" * 16) # trailing pad-shaped+ cases.append(bytes(base) + b"\x77" * 16) # trailing garbage+ mono = bytearray(base)+ mono[16 + 8 * 4] ^= 0x80 # offsets landmine+ cases.append(bytes(mono))+ dirty = bytearray(base)+ dirty[16 + (rf.N_STREAMS + 1) * 4] = 0x55 # header pad byte+ cases.append(bytes(dirty))++ for i, blob in enumerate(cases):+ c_cls, py_cls = c_parse(blob), py_parse(blob)+ self.assertEqual(c_cls, py_cls,+ msg=f"case {i}: C={c_cls} Python={py_cls}")+ self.assertEqual(c_parse(cases[0]), "OK") # control really is OK++ def test_verifier_never_crashes(self):+ """Hostile safetensors headers (the review round's traceback+ classes): zero tracebacks, nonzero exit, a named REFUSE line each."""+ name = "model.layers.0.mlp.experts.0.gate_proj.weight"+ ok_entry = {"dtype": "U8", "shape": [4], "data_offsets": [0, 4]}+ stamp = json.dumps({name: rf.FORMAT_NAME})+ cases = {+ "meta_not_string": {"__metadata__": {rf.METADATA_KEY: 5},+ name: dict(ok_entry)},+ "header_array": [1, 2, 3],+ "no_data_offsets": {"__metadata__": {rf.METADATA_KEY: stamp,+ rf.TABLE_KEY: "{}"},+ name: {"dtype": "U8", "shape": [4]}},+ "entry_not_dict": {"__metadata__": {rf.METADATA_KEY: stamp,+ rf.TABLE_KEY: "{}"},+ name: "hello"},+ "fmt_value_list": {"__metadata__":+ {rf.METADATA_KEY: json.dumps({name: ["a"]})},+ name: dict(ok_entry)},+ "reversed_offsets": {"__metadata__": {rf.METADATA_KEY: stamp,+ rf.TABLE_KEY: "{}"},+ name: {"dtype": "U8", "shape": [4],+ "data_offsets": [8, 2]}},+ "offsets_past_eof": {"__metadata__": {rf.METADATA_KEY: stamp,+ rf.TABLE_KEY: "{}"},+ name: {"dtype": "U8", "shape": [4],+ "data_offsets": [0, 4096]}},+ "offsets_not_pair": {"__metadata__": {rf.METADATA_KEY: stamp,+ rf.TABLE_KEY: "{}"},+ name: {"dtype": "U8", "shape": [4],+ "data_offsets": [1, 2, 3]}},+ }+ hostile = self.root / "hostile"+ hostile.mkdir()+ for label, hdr in cases.items():+ p = hostile / f"h_{label}.safetensors"+ hj = json.dumps(hdr, separators=(",", ":")).encode()+ with open(p, "wb") as f:+ f.write(len(hj).to_bytes(8, "little"))+ f.write(hj)+ f.write(b"\x00" * 32)+ r = run_tool("rans_verify.py", str(p))+ self.assertNotIn("Traceback", r.stderr,+ msg=f"{label}: crashed\n{r.stderr}")+ self.assertEqual(r.returncode, 1, msg=f"{label}: {r.stdout}")+ self.assertIn("REFUSE", r.stdout, msg=f"{label}: no REFUSE line")++ def test_partial_output_refusal(self):+ """An outdir already holding repack output is refused (crash+ evidence: an interrupted set must not be papered over); --force+ redoes it; a completed run leaves repack-manifest.json."""+ out = self.root / "partial"+ shards = self.repack(out)+ manifest = out / "repack-manifest.json"+ self.assertTrue(manifest.exists())+ man = json.loads(manifest.read_text())+ self.assertEqual(man["n_shards"], len(shards))++ # rerun over completed output: refused+ r = run_tool("repack_rans.py", "--indir", str(self.indir),+ "--outdir", str(out))+ self.assertEqual(r.returncode, 1)+ self.assertIn("refusing to overwrite", r.stderr)++ # simulate a crashed run: shards present, no completion manifest+ manifest.unlink()+ r = run_tool("repack_rans.py", "--indir", str(self.indir),+ "--outdir", str(out))+ self.assertEqual(r.returncode, 1)+ self.assertIn("NO manifest", r.stderr)++ # --force redoes from scratch, byte-identical to a fresh run+ r = run_tool("repack_rans.py", "--indir", str(self.indir),+ "--outdir", str(out), "--force")+ self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr)+ self.assertTrue(manifest.exists())+ fresh = self.repack(self.root / "partial_fresh")+ self.assertEqual(shards[0].read_bytes(), fresh[0].read_bytes())++ def test_quantize_freq_ties_and_termination(self):+ """Exact fractional-remainder ties must break identically everywhere+ (stable argsort -> lowest symbol index wins), and an impossible table+ size must raise instead of looping forever."""+ hist = np.zeros(16, dtype=np.int64)+ hist[1] = hist[2] = hist[3] = 1 # remainders tie exactly at 1/3+ freq = rf.quantize_freq(hist)+ self.assertEqual(freq.tolist(),+ [0, 5462, 5461, 5461, 0, 0, 0, 0,+ 0, 0, 0, 0, 0, 0, 0, 0])+ self.assertEqual(int(freq.sum()), rf.M)+ with self.assertRaises(ValueError):+ rf.quantize_freq(np.ones(16, dtype=np.int64), m=8)+if __name__ == "__main__":unittest.main()diff --git a/c/tools/rans_ctypes.c b/c/tools/rans_ctypes.cindex e316ad45..ce61e685 100644--- a/c/tools/rans_ctypes.c+++ b/c/tools/rans_ctypes.c@@ -15,6 +15,14 @@#define RC_EXPORT#endif+/* This bridge serves the int4-rans256-g0 TOOLS, so unlike the format-generic+ * header entry points it can and does pin n_streams to the format's 256 β+ * a zero or wrong-for-format value is refused by name before any division+ * or decode dispatch (the header entries additionally refuse zero). */+static int rc__nstreams_ok(uint32_t n_streams) {+ return n_streams == RANS_NSTREAMS;+}+/* Encode n nibbles into one chunk record. Returns the record length, or the* negated rans_err on failure. out_cap should come from rc_record_bound. */RC_EXPORT int64_t rc_record_encode(const uint8_t *nibbles, uint64_t n,@@ -22,6 +30,7 @@ RC_EXPORT int64_t rc_record_encode(const uint8_t *nibbles, uint64_t n,const uint16_t *slot_to_symbol,uint32_t scale_bits, uint32_t n_streams,uint8_t *out, uint64_t out_cap) {+ if (!rc__nstreams_ok(n_streams)) return -(int64_t)RANS_E_NSTREAMS;rans_table t;rans_err e = rans_table_init(&t, scale_bits, freq, start, slot_to_symbol);if (e != RANS_OK) return -(int64_t)e;@@ -33,6 +42,7 @@ RC_EXPORT int64_t rc_record_encode(const uint8_t *nibbles, uint64_t n,}RC_EXPORT uint64_t rc_record_bound(uint64_t n_symbols, uint32_t n_streams) {+ if (!rc__nstreams_ok(n_streams)) return 0; /* callers treat 0 as refusal */return rans_record_bound(n_symbols, n_streams);}@@ -41,6 +51,7 @@ RC_EXPORT uint64_t rc_record_bound(uint64_t n_symbols, uint32_t n_streams) {RC_EXPORT int32_t rc_record_parse(const uint8_t *blob, uint64_t blob_len,uint32_t n_streams,uint64_t *n_symbols, uint64_t *packed_bytes) {+ if (!rc__nstreams_ok(n_streams)) return (int32_t)RANS_E_NSTREAMS;rans_record rec;rans_err e = rans_record_parse(blob, blob_len, n_streams, &rec);if (e == RANS_OK) {@@ -61,6 +72,7 @@ RC_EXPORT int32_t rc_record_decode(const uint8_t *blob, uint64_t blob_len,const uint16_t *slot_to_symbol,uint32_t scale_bits, int32_t path,uint8_t *out_packed) {+ if (!rc__nstreams_ok(n_streams)) return (int32_t)RANS_E_NSTREAMS;rans_record rec;rans_err e = rans_record_parse(blob, blob_len, n_streams, &rec);if (e != RANS_OK) return (int32_t)e;diff --git a/c/tools/rans_format.py b/c/tools/rans_format.pyindex a90314f5..4c6e3614 100644--- a/c/tools/rans_format.py+++ b/c/tools/rans_format.py@@ -64,10 +64,25 @@ class RansRefusal(Exception):def quantize_freq(hist, m=M):"""hist: length-16 array of raw counts. Returns freq[16] uint32 summing tom; largest-remainder quantization; min 1 count for any nonzero-count- symbol (so it stays encodable); absent symbols get exactly 0."""+ symbol (so it stays encodable); absent symbols get exactly 0.++ Determinism: both argsorts use kind="stable" so exact fractional-remainder+ (or frequency) ties break by symbol index identically on every host and+ numpy version β contract 4's cross-host byte-identity depends on it.++ Termination: requires m >= the number of present symbols (raises+ ValueError otherwise β the min-1 constraint would be unsatisfiable).+ Given that, the deficit<0 loop provably terminates: while deficit < 0,+ sum(freq) = m - deficit > m >= n_present, so some entry exceeds 1 and+ every full pass over `order` decrements at least one entry."""hist = np.asarray(hist, dtype=np.float64)total = hist.sum()assert total > 0+ n_present = int(np.count_nonzero(hist))+ if m < n_present:+ raise ValueError(+ f"quantize_freq: m={m} < {n_present} present symbols β every "+ f"present symbol needs freq >= 1, so this table size is impossible")raw = hist / total * mfreq = np.floor(raw).astype(np.int64)nonzero = hist > 0@@ -76,7 +91,7 @@ def quantize_freq(hist, m=M):if deficit > 0:remainder = raw - np.floor(raw)remainder[~nonzero] = -1 # never give slots to symbols absent from data- order = np.argsort(-remainder)+ order = np.argsort(-remainder, kind="stable")i = 0while deficit > 0 and i < len(order):idx = order[i]@@ -85,7 +100,7 @@ def quantize_freq(hist, m=M):deficit -= 1i += 1elif deficit < 0:- order = np.argsort(-freq)+ order = np.argsort(-freq, kind="stable")i = 0while deficit < 0:idx = order[i % len(order)]@@ -317,8 +332,18 @@ def build_record(nibbles, freq, start, slot_to_symbol, scale_bits=SCALE_BITS,"""nibbles -> one chunk-record bytes object. Deterministic."""nib = np.ascontiguousarray(nibbles, dtype=np.uint8)n = nib.size+ if n_streams != N_STREAMS:+ raise RansRefusal("E_NSTREAMS",+ f"n_streams={n_streams} (this format fixes {N_STREAMS})")if n == 0:raise RansRefusal("E_EMPTY", "refusing to encode an empty tensor")+ # pre-scan parity with c/rans.h: refuse uncodable input by name before+ # any encoding work (values > 15, or symbols the table gives freq 0)+ freq_a = np.asarray(freq, dtype=np.uint32)+ if int(nib.max(initial=0)) > 15 or (freq_a[nib] == 0).any():+ raise RansRefusal("E_SYMBOL_UNCODABLE",+ "input contains a value > 15 or a symbol whose "+ "table frequency is zero")if LIB is not None:cap = int(LIB.rc_record_bound(n, n_streams)) + SLACKout = np.zeros(cap, dtype=np.uint8)@@ -345,13 +370,19 @@ def build_record(nibbles, freq, start, slot_to_symbol, scale_bits=SCALE_BITS,def parse_record(blob, n_streams=N_STREAMS):"""Validate framing; returns dict(n_symbols, packed_bytes, offsets,- payload). Raises RansRefusal with the same named classes as c/rans.h."""+ payload). Raises RansRefusal with the same named classes as c/rans.h β+ that parity is a TESTED property (the e2e suite feeds identical bytes to+ both implementations and asserts identical classes)."""+ if n_streams == 0:+ raise RansRefusal("E_NSTREAMS", "n_streams == 0")head = record_header_bytes(n_streams)if len(blob) < head:raise RansRefusal("E_TRUNCATED", f"{len(blob)} bytes < {head}-byte header")n_symbols, packed_bytes = struct.unpack_from("<QQ", blob, 0)if n_symbols == 0:raise RansRefusal("E_EMPTY", "n_symbols == 0")+ # Python ints do not wrap, so ceil(n/2) here already agrees with the C+ # side's non-wrapping n/2 + (n&1) for every uint64 value incl. UINT64_MAXif packed_bytes != (n_symbols + 1) // 2:raise RansRefusal("E_COUNT_MISMATCH",f"packed_bytes={packed_bytes} != ceil({n_symbols}/2)")@@ -372,6 +403,14 @@ def parse_record(blob, n_streams=N_STREAMS):raise RansRefusal("E_LENGTH_MISMATCH",f"blob is {len(blob)} bytes, framing derives "f"{head + _round16(payload_len)}")+ # amplification bound, identical to c/rans.h: no table this format admits+ # can encode more than payload_len*8*M_max symbols into this payload β+ # refuse the decompression bomb before anything sizes buffers from+ # n_symbols+ if n_symbols > payload_len * 8 * (1 << 15):+ raise RansRefusal("E_OVERSIZE",+ f"n_symbols={n_symbols} impossibly large for a "+ f"{payload_len}-byte payload")raw_head_end = 16 + (n_streams + 1) * 4if any(blob[raw_head_end:head]) or any(blob[head + payload_len:]):raise RansRefusal("E_PAD_NONZERO", "derived padding bytes are not zero")@@ -390,10 +429,18 @@ def decode_record(blob, table, n_streams=N_STREAMS, checked=True):slot = np.ascontiguousarray(table["slot_to_symbol"], dtype=np.uint16)sb = table["scale_bits"]n = rec["n_symbols"]+ # parse_record bounded n_symbols against the payload (E_OVERSIZE); a+ # record that legitimately passes can still be too big for THIS machine β+ # that is a named refusal, never an allocator traceback+ try:+ nib = np.empty(n, dtype=np.uint8)+ out = np.empty(rec["packed_bytes"], dtype=np.uint8)+ except MemoryError:+ raise RansRefusal("E_NOMEM",+ f"cannot allocate buffers for n_symbols={n}")if LIB is not None and not checked:buf = np.zeros(len(blob) + SLACK, dtype=np.uint8) # SLACK contractbuf[:len(blob)] = np.frombuffer(blob, dtype=np.uint8)- out = np.empty(rec["packed_bytes"], dtype=np.uint8)rc = LIB.rc_record_decode(_p8(buf), len(blob), n_streams, _p32(freq),_p32(start), _p16(slot), sb, -1, _p8(out))if rc != 0:@@ -402,7 +449,6 @@ def decode_record(blob, table, n_streams=N_STREAMS, checked=True):# checked per-stream decode (verification path), C-accelerated per streampayload = rec["payload"]offsets = rec["offsets"]- nib = np.empty(n, dtype=np.uint8)for i in range(n_streams):sub = payload[int(offsets[i]):int(offsets[i + 1])]n_i = len(range(i, n, n_streams))@@ -431,23 +477,58 @@ def decode_record(blob, table, n_streams=N_STREAMS, checked=True):# external safetensors dependency, same spirit as doctor.py's own parser# ---------------------------------------------------------------------------def read_header(path):+ """Parse AND validate a shard header. An untrusted container reaches the+ tools through this function, so the whole header shape is pinned here β+ every malformation is a named E_SHARD_MALFORMED refusal, never a+ downstream KeyError/TypeError traceback: the header must be a JSON+ object; every tensor entry an object with a string dtype, a list shape,+ and data_offsets = [b, e] with 0 <= b <= e <= data size; __metadata__+ (when present) a string-to-string object."""+ fsize = os.path.getsize(path)with open(path, "rb") as f:raw = f.read(8)if len(raw) != 8:raise RansRefusal("E_SHARD_MALFORMED", f"{path}: no header length")hlen = struct.unpack("<Q", raw)[0]- if hlen > 512 * 1024 * 1024:+ if hlen > 512 * 1024 * 1024 or 8 + hlen > fsize:raise RansRefusal("E_SHARD_MALFORMED", f"{path}: absurd header ({hlen} B)")try:header = json.loads(f.read(hlen))except ValueError as exc:raise RansRefusal("E_SHARD_MALFORMED", f"{path}: header not JSON: {exc}")- return header, 8 + hlen+ if not isinstance(header, dict):+ raise RansRefusal("E_SHARD_MALFORMED",+ f"{path}: header is {type(header).__name__}, not an object")+ data_start = 8 + hlen+ data_size = fsize - data_start+ for name, info in header.items():+ if name == "__metadata__":+ if not isinstance(info, dict) or \+ not all(isinstance(k, str) and isinstance(v, str)+ for k, v in info.items()):+ raise RansRefusal("E_SHARD_MALFORMED",+ f"{path}: __metadata__ is not a str->str object")+ continue+ if not isinstance(info, dict):+ raise RansRefusal("E_SHARD_MALFORMED",+ f"{path}:{name}: tensor entry is not an object")+ if not isinstance(info.get("dtype"), str) or \+ not isinstance(info.get("shape"), list):+ raise RansRefusal("E_SHARD_MALFORMED",+ f"{path}:{name}: missing/malformed dtype or shape")+ off = info.get("data_offsets")+ if (not isinstance(off, list) or len(off) != 2 or+ not all(isinstance(x, int) and not isinstance(x, bool) for x in off) or+ not (0 <= off[0] <= off[1] <= data_size)):+ raise RansRefusal("E_SHARD_MALFORMED",+ f"{path}:{name}: data_offsets {off!r} invalid for "+ f"{data_size} data bytes")+ return header, data_startdef read_tensor_bytes(path, header, data_start, name):info = header[name]- b, e = info["data_offsets"]+ b, e = info["data_offsets"] # validated by read_headerwith open(path, "rb") as f:f.seek(data_start + b)raw = f.read(e - b)diff --git a/c/tools/rans_verify.py b/c/tools/rans_verify.pyindex 234f8f92..efe692dd 100644--- a/c/tools/rans_verify.py+++ b/c/tools/rans_verify.py@@ -4,6 +4,15 @@ boundary. Accepts a spec-conformant shard; produces a NAMED refusal for everycorruption class; never crashes, never silently accepts.WHAT IS CHECKED, per shard:+ container layer:+ E_SHARD_MALFORMED unreadable/absurd header; header not a JSON+ object; tensor entries not objects; dtype/shape+ missing; data_offsets absent, non-[b,e],+ reversed, or past the end of the file;+ __metadata__ not a str->str object+ E_INTERNAL anything unexpected β a catch-all that converts+ any surprise into a named refusal so "never+ crashes" is a property, not a hopestamp layer (the stamp is MANDATORY for this format β entropy-coded sizesare data-dependent, so the stamp is the ONLY signal a U8 tensor isentropy-coded at all):@@ -22,9 +31,11 @@ WHAT IS CHECKED, per shard:E_TABLE_START start[] not the prefix sum of freq[]E_TABLE_SLOT slot_to_symbol inconsistent with freq[]/start[]E_TABLE_CRC slot-table bytes do not match the stamped crc32- record layer (per stamped tensor; same names as c/rans.h's rans_err):- E_TRUNCATED E_EMPTY E_COUNT_MISMATCH E_OFFSET_FIRST E_OFFSETS_MONOTONIC- E_STREAM_SHORT E_LENGTH_MISMATCH E_PAD_NONZERO+ record layer (per stamped tensor; same names as c/rans.h's rans_err, and+ the same-bytes/same-class parity with the C parser is a tested property):+ E_TRUNCATED E_EMPTY E_COUNT_MISMATCH E_OVERSIZE E_OFFSET_FIRST+ E_OFFSETS_MONOTONIC E_STREAM_SHORT E_LENGTH_MISMATCH E_PAD_NONZERO+ E_NOMEM (a record too large for this machine refuses by name)payload layer (per stream; a genuine encoder output ALWAYS satisfiesthese, so corruption is refused without any ground-truth bytes):E_STREAM_STATE_RANGE initial state outside the encoder's invariant@@ -61,6 +72,8 @@ def refuse(failures, shard, tensor, code, detail):def verify_shard(path, failures, max_tensors=None):+ """One shard. Wrapped by main()'s catch-all so no input can escape as a+ traceback; rf.read_header pins the whole header shape by name first."""try:header, data_start = rf.read_header(path)except rf.RansRefusal as exc:@@ -111,37 +124,31 @@ def verify_shard(path, failures, max_tensors=None):f"--max-tensors {max_tensors} of {len(ours)} stamped tensors")breakchecked += 1- if name not in header:- refuse(failures, path, name, "E_STAMP_DANGLING",- "stamped but not present in this shard")- continue- info = header[name]- if info["dtype"] != "U8":- refuse(failures, path, name, "E_STAMP_DTYPE",- f"dtype {info['dtype']} != U8")- continuetry:+ if name not in header:+ raise rf.RansRefusal("E_STAMP_DANGLING",+ "stamped but not present in this shard")+ info = header[name]+ if info["dtype"] != "U8":+ raise rf.RansRefusal("E_STAMP_DTYPE",+ f"dtype {info['dtype']} != U8")blob, _ = rf.read_tensor_bytes(path, header, data_start, name)- except rf.RansRefusal as exc:- refuse(failures, path, name, exc.code, exc.detail)- continue- try:packed = rf.decode_record(blob, table)- except rf.RansRefusal as exc:- refuse(failures, path, name, exc.code, exc.detail)- continue- # deterministic re-encode cross-check- try:+ # deterministic re-encode cross-checkre_rec = rf.build_record(rf.unpack_nibbles(packed), table["freq"],table["start"], table["slot_to_symbol"],table["scale_bits"])+ if re_rec != blob:+ raise rf.RansRefusal("E_REENCODE_MISMATCH",+ "re-encoding the decoded stream does not "+ "reproduce the record β content corruption")except rf.RansRefusal as exc:refuse(failures, path, name, exc.code, exc.detail)continue- if re_rec != blob:- refuse(failures, path, name, "E_REENCODE_MISMATCH",- "re-encoding the decoded stream does not reproduce the "- "record β content corruption")+ except Exception as exc: # noqa: BLE001 β the point+ # anything unexpected becomes a named refusal: "never crashes"+ # is enforced here, not promised+ refuse(failures, path, name, "E_INTERNAL", repr(exc))continuetrusted += 1print(f"[trust] {os.path.basename(path)}: {trusted}/{checked} verified "@@ -162,7 +169,10 @@ def main():if not os.path.exists(p):refuse(failures, p, None, "E_SHARD_MALFORMED", "no such file")continue- verify_shard(p, failures, a.max_tensors)+ try:+ verify_shard(p, failures, a.max_tensors)+ except Exception as exc: # noqa: BLE001 β the point+ refuse(failures, p, None, "E_INTERNAL", repr(exc))if failures:print(f"RESULT: REFUSE ({len(failures)} refusal(s))")sys.exit(1)diff --git a/c/tools/repack_rans.py b/c/tools/repack_rans.pyindex 92eb304c..df87dd7d 100644--- a/c/tools/repack_rans.py+++ b/c/tools/repack_rans.py@@ -59,10 +59,20 @@ import rans_format as rf # noqa: E402EXPERT_RE = re.compile(r"^model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate|up|down)_proj\.weight$")-# Per-row geometry guard (see module docstring). MIN_ROW_RATIO = I/2 for the-# smallest hidden dim this tool accepts; the load-bearing exclusion is 32,-# the exact ratio EVERY per-group-64 container measures regardless of shape.+# Per-row geometry guard (see module docstring). When the weight tensor+# carries a 2-D [O, rb] shape, per-row geometry is verified POSITIVELY+# (qs elements == O). Flattened 1-D containers (what convert_fp8_to_int4.py+# emits) can only be judged by the weight-bytes/scale-rows ratio:+# per-row -> ratio == I/2 (3072 / 1024 for real GLM projections)+# per-group-64 -> ratio == exactly 32, ALWAYS, regardless of shape+# per-group-128 (upstream fmt=4's actual gs) -> ratio == exactly 64+# so 32 refuses as g64, 64 refuses as AMBIGUOUS (indistinguishable from a+# per-row I=128 tensor by bytes alone), and anything below MIN_ROW_RATIO+# refuses as not-per-row. Collateral: 1-D per-row tensors with I <= 128 are+# refused β loud, by name, and vacuous for every real container this repo+# targets.G64_RATIO = 32+G128_RATIO = 64MIN_ROW_RATIO = 64@@ -110,6 +120,18 @@ def select_targets(index, layers):raise rf.RansRefusal("E_GEOMETRY_NOT_PER_ROW",f"{name}: {qs} is {sb} bytes (not F32-shaped)")rows = sb // 4+ wshape = info["shape"]+ if len(wshape) == 2:+ # positive per-row verification: [O, rb] weight demands exactly+ # one scale per row β no ratio heuristics involved+ if rows != wshape[0]:+ raise rf.RansRefusal(+ "E_GEOMETRY_NOT_PER_ROW",+ f"{name}: weight shape {wshape} has {wshape[0]} rows but "+ f"{qs} carries {rows} scales β not per-row geometry")+ selected.append(name)+ continue+ # flattened 1-D container: byte-ratio judgement (docstring above)if wb % rows:raise rf.RansRefusal("E_GEOMETRY_NOT_PER_ROW",@@ -121,6 +143,14 @@ def select_targets(index, layers):f"{name}: weight/scale ratio {ratio} is the per-group-64 "f"signature β g64 containers are out of this format's v1 scope "f"(materially weaker entropy payoff; needs its own round)")+ if ratio == G128_RATIO:+ raise rf.RansRefusal(+ "E_GEOMETRY_AMBIGUOUS",+ f"{name}: weight/scale ratio {ratio} is exactly the "+ f"per-group-128 signature (upstream grouped-int4's gs) and "+ f"indistinguishable by bytes from a per-row I=128 tensor β "+ f"refusing rather than stamping per-row geometry it cannot "+ f"prove")if ratio < MIN_ROW_RATIO:raise rf.RansRefusal("E_GEOMETRY_NOT_PER_ROW",@@ -141,6 +171,10 @@ def main():help="comma-separated layer subset (default: every layer)")ap.add_argument("--dry-run", action="store_true",help="inventory only: scan headers, print selection, write nothing")+ ap.add_argument("--force", action="store_true",+ help="overwrite an outdir that already contains repack output "+ "(default: refuse β a pre-existing partial set from an "+ "interrupted run must never be silently papered over)")a = ap.parse_args()layers = None@@ -195,9 +229,29 @@ def main():print(f"[table] pooled histogram {hist.tolist()}")print(f"[table] freq {freq.tolist()} (M={rf.M}, scale_bits={rf.SCALE_BITS})")- # pass 2: encode + verify + write, one output shard per input shard+ # pass 2: encode + verify + write, one output shard per input shard.+ # CRASH EVIDENCE: refuse a non-empty output directory (a crashed earlier+ # run leaves individually-valid shards with no marker that the SET is+ # incomplete β rerunning over them silently would hide that), and write+ # repack-manifest.json only after every shard landed, so "manifest+ # present and matching" == "the set is complete".os.makedirs(a.outdir, exist_ok=True)+ manifest_path = os.path.join(a.outdir, "repack-manifest.json")+ pre_existing = sorted(glob.glob(os.path.join(a.outdir, "out-rans-*.safetensors")))+ if (pre_existing or os.path.exists(manifest_path)) and not a.force:+ print(f"ERROR: {a.outdir} already contains repack output "+ f"({len(pre_existing)} shard(s)"+ f"{', manifest present' if os.path.exists(manifest_path) else ', NO manifest β likely an interrupted run'}) "+ f"β refusing to overwrite; pass --force to redo from scratch",+ file=sys.stderr)+ sys.exit(1)+ if a.force:+ for p in pre_existing:+ os.remove(p)+ if os.path.exists(manifest_path):+ os.remove(manifest_path)out_n = 0+ shard_manifest = []total_record = 0for p in sorted(by_shard):tensors = []@@ -226,16 +280,37 @@ def main():}out_path = os.path.join(a.outdir, f"out-rans-{out_n:05d}.safetensors")rf.write_shard(out_path, tensors, metadata)+ shard_manifest.append({+ "file": os.path.basename(out_path),+ "source": os.path.basename(p),+ "bytes": os.path.getsize(out_path),+ "weight_tensors": len(fmt_map),+ })print(f"[write] {out_path}: {len(tensors)} tensor(s) "f"({os.path.getsize(out_path) / 1e9:.4f} GB) "f"from {os.path.basename(p)}")out_n += 1+ # completion marker, written LAST and deterministically (no timestamps):+ # its absence over a shard set means the producing run did not finish+ manifest = {+ "format": rf.FORMAT_NAME,+ "table_crc32": json.loads(table_json)["table_crc32"],+ "n_shards": out_n,+ "n_weight_tensors": len(selected),+ "shards": shard_manifest,+ }+ tmp = manifest_path + ".tmp"+ with open(tmp, "w") as fh:+ json.dump(manifest, fh, indent=1, sort_keys=True)+ os.replace(tmp, manifest_path)+ratio = total_record / total_wprint(f"[ratio] record/original: {ratio:.4f} ({(1 - ratio) * 100:.2f}% "f"reduction incl. framing), {total_w / 1e6:.1f} MB -> "f"{total_record / 1e6:.1f} MB")- print(f"[done] {out_n} shard(s), every record round-trip-verified byte-exact")+ print(f"[done] {out_n} shard(s) + repack-manifest.json, every record "+ f"round-trip-verified byte-exact")if __name__ == "__main__":diff --git a/docs/int4-rans256-g0.md b/docs/int4-rans256-g0.mdindex 328d0684..96df21c9 100644--- a/docs/int4-rans256-g0.md+++ b/docs/int4-rans256-g0.md@@ -38,7 +38,12 @@ second codec path in v1).### Chunk record (one record = one weight tensor's bytes)-All integers little-endian. `N` = 256 streams.+All integers little-endian β explicitly including the `n_symbols` and+`packed_bytes` u64 fields and every `stream_offsets` u32 (the same+byte order the base64 `slot_to_symbol` field spells out). `N` = 256+streams. The reference C implementation reads/writes these with+native-order accesses and therefore refuses to compile on a big-endian+host rather than emit byte-swapped records.```offset 0: n_symbols u64 -- nibble count for this tensor@@ -60,7 +65,24 @@ derived value. Validators refuse nonzero padding bytes. Alignment is**record-relative**, not file-absolute: safetensors does not 16-aligntensor offsets, so a consumer that wants aligned u32/u64 access must read ormap the tensor's bytes into an aligned allocation (any `malloc`/GPU-buffer-allocation qualifies).+allocation qualifies; the reference parser refuses a buffer that is not at+least 4-byte aligned rather than perform unaligned accesses).++Validity rules every conformant record satisfies (and validators enforce):++- `packed_bytes == ceil(n_symbols / 2)`, computed in a non-wrapping form+ (`n/2 + (n & 1)`) so `n_symbols = 2^64 - 1` cannot forge a match;+- `stream_offsets[0] == 0`, offsets non-decreasing, and **every stream is at+ least 4 bytes** β even a stream that encodes zero symbols carries its+ 4 flushed state bytes;+- the record's total length equals the derived framing exactly (no trailing+ bytes), and every derived padding byte is zero;+- **amplification bound**: no admissible table can encode more than+ `payload_len * 8 * M_max` symbols into a payload (`M_max = 2^15`, the+ format's largest table size β a symbol's cost is bounded below by+ `log2(M / (M-1))` bits). A header claiming more `n_symbols` than that is+ a decompression bomb and must be refused *before* anything sizes buffers+ from it.### Interleaving@@ -80,7 +102,18 @@ design): 32-bit state, byte renormalization, `L = 2^23`,`scale_bits = 14` (`M = 16384`).- encode walks the input backwards, writes bytes backwards, flushes the- final 4-byte state; decode reads forward:+ final 4-byte state. Per symbol `s` (this is the complete recipe a+ bit-identical third-party **writer** needs): with the state+ `x` (initialized to `L`), first renormalize by emitting low bytes β+ `x_max = ((L >> scale_bits) << 8) * freq[s]`;+ `while (x >= x_max) { emit byte x & 0xFF; x >>= 8; }` β then update+ `x = ((x / freq[s]) << scale_bits) + (x % freq[s]) + start[s]`+ (integer division). After the last (i.e. first-in-stream-order) symbol,+ emit the remaining 32-bit state as 4 bytes, low byte last. The emitted+ byte sequence, reversed into forward order, is the stream. Worst case a+ `freq = 1` symbol emits `scale_bits` bits, so a stream of `n` symbols+ never exceeds `ceil(n * scale_bits / 8) + 4` bytes;+- decode reads forward:`x = 4 bytes big-endian`, then per symbol:`slot = x & (M-1)`; `s = slot_to_symbol[slot]`;`x = freq[s]*(x >> scale_bits) + slot - start[s]`;