Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

rans: harden the edges the review round reached β€” parse bounds, encoder bound, never-crash validator

monotophic4dc19b344788
diff --git a/c/.gitignore b/c/.gitignore
index 3299f514..b77dca5a 100644
--- a/c/.gitignore
+++ b/c/.gitignore
@@ -11,3 +11,5 @@ tools/iq3.dll
tools/librans_c.so
tools/librans_c.dylib
tools/rans_c.dll
+tests/fuzz_rans
+tests/*.dSYM/
diff --git a/c/Makefile b/c/Makefile
index b629d8ea..9247ab81 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -460,6 +460,18 @@ tests/test_e8_kernel$(EXE): tests/test_e8_kernel.c quant.h
tests/test_rans$(EXE): tests/test_rans.c rans.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+# Structured-mutation fuzz for the rans record parser/decoder under
+# ASan+UBSan β€” seeded and bounded (a few thousand mutants, well under a
+# minute), asserting no sanitizer report and byte identity across every
+# compiled decode arm on every accepted record. NOT a test gate (sanitizer
+# flags differ from the normal build); build+run on demand:
+# make fuzz-rans
+fuzz-rans: tests/fuzz_rans.c rans.h
+ $(CC) -O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all \
+ -Wall -Wextra -Wno-unused-parameter -Wno-unused-function \
+ tests/fuzz_rans.c -o tests/fuzz_rans -lm
+ ./tests/fuzz_rans
+
tests/test_int3$(EXE): tests/test_int3.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
@@ -505,7 +517,12 @@ tests/test_pipe_block$(EXE): tests/test_pipe_block.c colibri.c st.h uring.h json
test-c: $(TEST_BINS)
$(PYTHON) tools/run_tests.py $(TEST_BINS)
-test-python:
+# test-python builds the rans ctypes bridge first: without it the repack e2e
+# test's C-vs-Python byte-identity pin would silently compare Python against
+# itself (the test ALSO skips loudly if the library is somehow still absent,
+# e.g. when invoked via unittest directly β€” but a green `make test` must mean
+# the pin was actually checked).
+test-python: $(RANSLIB)
$(PYTHON) -m unittest discover -s tests -p 'test_*.py'
test: test-c test-python
@@ -567,4 +584,4 @@ clean:
bench: iobench$(EXE)
@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) β€” run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi
-.PHONY: all colibri glm iq3 rans cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
+.PHONY: all colibri glm iq3 rans fuzz-rans cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
diff --git a/c/rans.h b/c/rans.h
index 0d355973..f0aaad81 100644
--- a/c/rans.h
+++ b/c/rans.h
@@ -72,6 +72,15 @@
#include <string.h>
#include <stdint.h>
+/* ENDIANNESS PRECONDITION: record header integers (n_symbols, packed_bytes,
+ * stream_offsets) are little-endian on the wire, and this implementation
+ * reads/writes them with native-order memcpy β€” correct only on little-endian
+ * hosts. Refuse to compile elsewhere rather than emit byte-swapped records
+ * that silently break the cross-host byte-identity contract. */
+#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__)
+#error "rans.h assumes a little-endian host (record integers are little-endian)"
+#endif
+
#if defined(__AVX512F__) && defined(__AVX512BW__)
#include <immintrin.h>
#endif
@@ -83,6 +92,7 @@
#define RANS_ALPHABET 16
#define RANS_NSTREAMS 256 /* int4-rans256-g0's stream count */
#define RANS_SLACK 64 /* readable bytes required past record buffers */
+#define RANS_SCALE_BITS_MAX 15 /* rans_table_init's upper bound on scale_bits */
/* ---- scalar stream codec (ported unmodified from the proven prototype) --- */
@@ -98,7 +108,11 @@ static size_t rans_encode_stream(const uint8_t *nibbles, size_t n,
uint32_t x = RANS_L;
uint8_t *ptr = out_buf + out_cap; /* write backwards from the end */
for (size_t i = n; i-- > 0; ) {
- uint32_t s = nibbles[i];
+ /* mask to the nibble alphabet: an out-of-range input byte must never
+ * index past freq[16]/start[16] (record-level entry points refuse
+ * such input by name BEFORE encoding; the mask makes this primitive
+ * memory-safe for any bytes regardless) */
+ uint32_t s = nibbles[i] & 15u;
uint32_t f = freq[s];
uint32_t st = start[s];
/* renormalize: keep x below x_max so the update keeps x in [L, L*256) */
@@ -163,11 +177,14 @@ typedef enum {
RANS_E_TRUNCATED, /* blob shorter than its own framing claims */
RANS_E_EMPTY, /* n_symbols == 0 */
RANS_E_COUNT_MISMATCH, /* packed_bytes != ceil(n_symbols/2) */
+ RANS_E_OVERSIZE, /* n_symbols impossibly large for payload */
RANS_E_OFFSET_FIRST, /* stream_offsets[0] != 0 */
RANS_E_OFFSETS_MONOTONIC, /* stream_offsets not non-decreasing */
RANS_E_STREAM_SHORT, /* a stream shorter than its 4-byte state */
RANS_E_LENGTH_MISMATCH, /* blob length != derived framing length */
RANS_E_PAD_NONZERO, /* a derived padding byte is not zero */
+ RANS_E_UNALIGNED, /* record buffer not 4-byte aligned */
+ RANS_E_NSTREAMS, /* n_streams zero / wrong for the format */
/* table-level refusals (rans_table_init) */
RANS_E_TABLE_SCALE, /* scale_bits out of range (1..15) */
RANS_E_TABLE_FREQ_SUM, /* freq[] does not sum to M */
@@ -178,6 +195,11 @@ typedef enum {
RANS_E_STREAM_UNDERRUN, /* fewer than 4 bytes for the initial state */
RANS_E_STREAM_LEFTOVER, /* decode finished with bytes unconsumed */
RANS_E_STREAM_FINAL_STATE, /* final state != L (not an encoder output) */
+ /* encode-side refusals */
+ RANS_E_SYMBOL_UNCODABLE, /* input contains a value > 15 or a symbol
+ whose table frequency is zero */
+ RANS_E_SCRATCH, /* encoder scratch/output bound exhausted
+ (a bound failure, NOT a memory condition) */
/* misc */
RANS_E_NOMEM,
RANS_E_PATH_UNAVAILABLE /* RANS_PATH forced an arm this build/CPU/env
@@ -190,11 +212,16 @@ static const char *rans_err_name(rans_err e) {
case RANS_E_TRUNCATED: return "E_TRUNCATED";
case RANS_E_EMPTY: return "E_EMPTY";
case RANS_E_COUNT_MISMATCH: return "E_COUNT_MISMATCH";
+ case RANS_E_OVERSIZE: return "E_OVERSIZE";
case RANS_E_OFFSET_FIRST: return "E_OFFSET_FIRST";
case RANS_E_OFFSETS_MONOTONIC: return "E_OFFSETS_MONOTONIC";
case RANS_E_STREAM_SHORT: return "E_STREAM_SHORT";
case RANS_E_LENGTH_MISMATCH: return "E_LENGTH_MISMATCH";
case RANS_E_PAD_NONZERO: return "E_PAD_NONZERO";
+ case RANS_E_UNALIGNED: return "E_UNALIGNED";
+ case RANS_E_NSTREAMS: return "E_NSTREAMS";
+ case RANS_E_SYMBOL_UNCODABLE: return "E_SYMBOL_UNCODABLE";
+ case RANS_E_SCRATCH: return "E_SCRATCH";
case RANS_E_TABLE_SCALE: return "E_TABLE_SCALE";
case RANS_E_TABLE_FREQ_SUM: return "E_TABLE_FREQ_SUM";
case RANS_E_TABLE_START: return "E_TABLE_START";
@@ -306,38 +333,54 @@ static uint64_t rans_record_header_bytes(uint32_t n_streams) {
return rans_round16(16u + ((uint64_t)n_streams + 1u) * 4u);
}
-/* Worst-case record size for n nibbles: per-stream worst case mirrors the
- * encoder's own scratch rule (n + n/2 + 64 per stream is generous for any
- * 16-symbol table whose present symbols have freq >= 1), plus framing. */
+/* Worst-case record size for n nibbles. A symbol with table frequency f
+ * costs at most log2(M/f) bits, so the per-stream worst case over any valid
+ * table (f >= 1) is scale_bits bits/symbol β€” 1.875 bytes/symbol at the
+ * maximum scale_bits of 15 β€” plus the 4 flushed state bytes. This bound
+ * uses RANS_SCALE_BITS_MAX so it is table-independent and safe for every
+ * table rans_table_init accepts (the old "n + n/2" rule was FALSE for
+ * freq=1 symbols at scale_bits >= 13). Returns 0 for n_streams == 0. */
static uint64_t rans_record_bound(uint64_t n_symbols, uint32_t n_streams) {
+ if (n_streams == 0) return 0;
uint64_t per = n_symbols / n_streams + 1u;
+ uint64_t per_bytes = (per * RANS_SCALE_BITS_MAX + 7u) / 8u + 4u + 64u;
return rans_record_header_bytes(n_streams) +
- rans_round16((per + per / 2u + 64u) * n_streams);
+ rans_round16(per_bytes * n_streams);
}
/* Encode n nibbles as one int4-rans256-g0 chunk record (n_streams round-robin
* interleaved streams, shared table). Writes at most rans_record_bound()
* bytes into out; *out_len receives the exact record length. Deterministic:
- * same input + same table => byte-identical output. Returns RANS_OK, or
- * RANS_E_EMPTY / RANS_E_NOMEM. */
+ * same input + same table => byte-identical output. Returns RANS_OK, or:
+ * RANS_E_EMPTY (n == 0), RANS_E_NSTREAMS (n_streams == 0),
+ * RANS_E_SYMBOL_UNCODABLE (an input value > 15, or a symbol whose table
+ * frequency is zero β€” detected by a pre-scan BEFORE any encoding work),
+ * RANS_E_SCRATCH (a codec bound was exceeded β€” a logic/bound failure, never
+ * reported as a memory condition), RANS_E_NOMEM (malloc failed). */
static rans_err rans_record_encode(const uint8_t *nibbles, uint64_t n,
const rans_table *t, uint32_t n_streams,
uint8_t *out, uint64_t out_cap,
uint64_t *out_len) {
+ if (n_streams == 0) return RANS_E_NSTREAMS;
if (n == 0) return RANS_E_EMPTY;
+ /* pre-scan: refuse uncodable input by name, before touching `out` */
+ for (uint64_t j = 0; j < n; j++)
+ if (nibbles[j] > 15u || t->freq[nibbles[j]] == 0)
+ return RANS_E_SYMBOL_UNCODABLE;
uint64_t head = rans_record_header_bytes(n_streams);
- if (out_cap < head) return RANS_E_NOMEM;
+ if (out_cap < head) return RANS_E_SCRATCH;
uint64_t sub_max = n / n_streams + 1u;
- uint64_t scap = sub_max + sub_max / 2u + 64u; /* per-stream scratch cap */
+ /* per-stream scratch: a freq>=1 symbol costs at most scale_bits bits, so
+ * ceil(sub_max*scale_bits/8) + 4 flush bytes (+ slack) always fits */
+ uint64_t scap = (sub_max * t->scale_bits + 7u) / 8u + 4u + 64u;
uint8_t *sub = (uint8_t *)malloc(sub_max ? sub_max : 1);
uint8_t *enc = (uint8_t *)malloc(scap);
if (!sub || !enc) { free(sub); free(enc); return RANS_E_NOMEM; }
memset(out, 0, head);
- uint64_t packed_bytes = (n + 1u) / 2u;
+ uint64_t packed_bytes = n / 2u + (n & 1u);
memcpy(out, &n, 8);
memcpy(out + 8, &packed_bytes, 8);
- uint32_t *offs = (uint32_t *)(out + 16);
uint64_t pos = 0; /* payload cursor */
uint8_t *payload = out + head;
@@ -346,17 +389,19 @@ static rans_err rans_record_encode(const uint8_t *nibbles, uint64_t n,
for (uint64_t j = i; j < n; j += n_streams) sub[ns++] = nibbles[j];
size_t off = rans_encode_stream(sub, ns, t->freq, t->start,
t->scale_bits, enc, scap);
- if (off == (size_t)-1) { free(sub); free(enc); return RANS_E_NOMEM; }
+ if (off == (size_t)-1) { free(sub); free(enc); return RANS_E_SCRATCH; }
uint64_t len = scap - off;
- if (head + pos + len > out_cap) { free(sub); free(enc); return RANS_E_NOMEM; }
+ if (head + pos + len > out_cap) { free(sub); free(enc); return RANS_E_SCRATCH; }
memcpy(payload + pos, enc + off, len);
- offs[i] = (uint32_t)pos;
+ uint32_t pos32 = (uint32_t)pos;
+ memcpy(out + 16 + (uint64_t)i * 4u, &pos32, 4);
pos += len;
- if (pos > 0xFFFFFFFFu) { free(sub); free(enc); return RANS_E_NOMEM; }
+ if (pos > 0xFFFFFFFFu) { free(sub); free(enc); return RANS_E_SCRATCH; }
}
- offs[n_streams] = (uint32_t)pos;
+ uint32_t total32 = (uint32_t)pos;
+ memcpy(out + 16 + (uint64_t)n_streams * 4u, &total32, 4);
uint64_t total = head + rans_round16(pos);
- if (total > out_cap) { free(sub); free(enc); return RANS_E_NOMEM; }
+ if (total > out_cap) { free(sub); free(enc); return RANS_E_SCRATCH; }
memset(payload + pos, 0, rans_round16(pos) - pos);
*out_len = total;
free(sub); free(enc);
@@ -376,30 +421,53 @@ typedef struct {
/* Parse + validate one record blob (a tensor's full byte range). TRUST-
* VERIFY-REFUSE: every malformation class returns its own named error and
* the record is unusable on failure; there is no partial acceptance. The
- * blob is borrowed. NOTE for the batched decode arms: the blob must satisfy
- * the RANS_SLACK contract (header comment) β€” parsing alone does not read
- * past blob_len. */
+ * blob is borrowed and must be 4-byte aligned (any malloc'd or numpy-backed
+ * buffer qualifies; refused by name otherwise) so the stream_offsets
+ * pointer handed to the decode kernels indexes with defined behavior.
+ * NOTE for the batched decode arms: the blob must additionally satisfy the
+ * RANS_SLACK contract (header comment) β€” parsing alone does not read past
+ * blob_len, and it performs no allocation (in particular nothing
+ * proportional to the untrusted n_symbols field). */
static rans_err rans_record_parse(const uint8_t *blob, uint64_t blob_len,
uint32_t n_streams, rans_record *out) {
memset(out, 0, sizeof(*out));
+ if (n_streams == 0) return RANS_E_NSTREAMS;
+ if (((uintptr_t)blob & 3u) != 0) return RANS_E_UNALIGNED;
uint64_t head = rans_record_header_bytes(n_streams);
if (blob_len < head) return RANS_E_TRUNCATED;
uint64_t n_symbols, packed_bytes;
memcpy(&n_symbols, blob, 8);
memcpy(&packed_bytes, blob + 8, 8);
if (n_symbols == 0) return RANS_E_EMPTY;
- if (packed_bytes != (n_symbols + 1u) / 2u) return RANS_E_COUNT_MISMATCH;
- const uint32_t *offs = (const uint32_t *)(blob + 16);
- if (offs[0] != 0) return RANS_E_OFFSET_FIRST;
+ /* non-wrapping form of packed_bytes == ceil(n_symbols/2): the additive
+ * form (n_symbols+1)/2 wraps at UINT64_MAX and would accept
+ * packed_bytes == 0 for it */
+ if (packed_bytes != n_symbols / 2u + (n_symbols & 1u))
+ return RANS_E_COUNT_MISMATCH;
+ uint32_t off_prev, off_cur;
+ memcpy(&off_prev, blob + 16, 4);
+ if (off_prev != 0) return RANS_E_OFFSET_FIRST;
for (uint32_t i = 0; i < n_streams; i++) {
- if (offs[i + 1] < offs[i]) return RANS_E_OFFSETS_MONOTONIC;
+ memcpy(&off_cur, blob + 16 + ((uint64_t)i + 1u) * 4u, 4);
+ if (off_cur < off_prev) return RANS_E_OFFSETS_MONOTONIC;
/* every stream carries at least its 4-byte flushed state (true even
* for streams that encode zero symbols) */
- if (offs[i + 1] - offs[i] < 4u) return RANS_E_STREAM_SHORT;
+ if (off_cur - off_prev < 4u) return RANS_E_STREAM_SHORT;
+ off_prev = off_cur;
}
- uint64_t payload_len = offs[n_streams];
+ uint64_t payload_len = off_prev; /* == offsets[n_streams] */
if (head + payload_len > blob_len) return RANS_E_TRUNCATED;
if (blob_len != head + rans_round16(payload_len)) return RANS_E_LENGTH_MISMATCH;
+ /* amplification bound: a symbol costs at least log2(M/(M-1)) bits under
+ * any valid table (present symbols of a genuine record cost more; the
+ * degenerate freq==M single-symbol table costs ~0 per symbol but still
+ * satisfies this bound easily). n_symbols beyond payload_len*8*M_max is
+ * impossible for ANY table this format admits β€” refuse the
+ * decompression bomb here, before any consumer sizes buffers from
+ * n_symbols. (No overflow: payload_len < 2^32, M_max = 2^15.) */
+ if (n_symbols > payload_len * 8u * (uint64_t)(1u << RANS_SCALE_BITS_MAX))
+ return RANS_E_OVERSIZE;
+ const uint32_t *offs = (const uint32_t *)(blob + 16); /* aligned: checked */
/* derived padding must be zero: bits hiding in the pad would make two
* "identical" containers differ and defeat writer determinism */
for (uint64_t i = 16u + ((uint64_t)n_streams + 1u) * 4u; i < head; i++)
@@ -499,6 +567,40 @@ static int rans__selftest_arm(rans_path p) {
rans_record_encode(data, NSYM, &t, NT, rec_buf, cap, &rec_len) == RANS_OK) {
rans_record rec;
if (rans_record_parse(rec_buf, rec_len, NT, &rec) == RANS_OK) {
+ /* MACHINE-CHECKED band coverage: the load-bearing property of
+ * this fixture is that it drives multi-byte renorm refills
+ * (post-update x < 2^15 => refill count kb == 2) β€” the band a
+ * comfortable table never visits and where a vector-arm renorm
+ * bug hides. Count kb with the oracle-shaped loop and FAIL the
+ * selftest if the fixture ever stops covering it, instead of
+ * trusting a comment. (kb == 3 needs x < 2^7, impossible
+ * mid-stream: post-update x >= L >> scale_bits >= 2^8.) */
+ uint64_t kb2 = 0;
+ for (uint32_t i = 0; i < NT; i++) {
+ const uint8_t *ptr = rec.payload + rec.stream_offsets[i];
+ const uint8_t *e2 = rec.payload + rec.stream_offsets[i + 1];
+ uint32_t x = 0;
+ for (int b2 = 0; b2 < 4; b2++) x = (x << 8) | *ptr++;
+ for (uint64_t j = i; j < rec.n_symbols; j += NT) {
+ uint32_t sl2 = x & (M - 1u);
+ uint32_t sy2 = slot[sl2];
+ x = freq[sy2] * (x >> sb) + sl2 - start[sy2];
+ uint32_t kb = 0;
+ while (x < RANS_L) {
+ if (ptr >= e2) break;
+ x = (x << 8) | *ptr++;
+ kb++;
+ }
+ if (kb >= 2) kb2++;
+ }
+ }
+ if (kb2 == 0) {
+ fprintf(stderr, "[rans] selftest fixture lost multi-byte "
+ "renorm coverage β€” fixture bug, arm not trusted\n");
+ free(rec_buf);
+ rans_table_free(&t);
+ return 0;
+ }
memset(packed_ref, 0, sizeof(packed_ref));
for (int i = 0; i < NSYM; i++)
packed_ref[i >> 1] |= (uint8_t)(data[i] << ((i & 1) * 4));
@@ -1015,6 +1117,9 @@ static void rans_record_decode_groups(const rans_record *rec, const rans_table *
static rans_err rans_record_decode_packed(const rans_record *rec, const rans_table *t,
uint32_t n_streams, rans_path p,
uint8_t *out_packed) {
+ /* n_streams == 0 would decode zero streams and return uninitialized
+ * scratch as "successful" output β€” refuse before any path can run */
+ if (n_streams == 0) return RANS_E_NSTREAMS;
if (p == RANS_PATH_INVALID) return RANS_E_PATH_UNAVAILABLE;
/* an arm this build did not compile refuses rather than silently running
* scalar: a quiet downgrade would let "we tested <arm>" become a false
diff --git a/c/tests/fuzz_rans.c b/c/tests/fuzz_rans.c
new file mode 100644
index 00000000..10daa303
--- /dev/null
+++ b/c/tests/fuzz_rans.c
@@ -0,0 +1,177 @@
+/* Structured-mutation fuzz for rans.h parse+decode against hostile records.
+ * Every mutant blob is placed in a fresh heap buffer of exactly
+ * blob_len + RANS_SLACK bytes so ASan catches any read past the documented
+ * slack contract. Decode output buffer is exactly packed_bytes.
+ *
+ * Seeded and bounded (7 base records x 700 structured mutants). Build+run
+ * via `make fuzz-rans` (ASan+UBSan, non-recoverable). Exit nonzero on any
+ * cross-arm divergence; sanitizers abort on any memory/UB report. NOT part
+ * of TEST_BINS: the sanitizer flags differ from the normal test build.
+ * Adopted from the E1 review round's audit harness (its first run found the
+ * u64-wrap acceptance in rans_record_parse that E_COUNT_MISMATCH's
+ * non-wrapping form now refuses). */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include "../rans.h"
+
+static uint64_t rng_state = 0x9E3779B97F4A7C15ULL;
+static uint32_t rnd(void) {
+ rng_state ^= rng_state << 13; rng_state ^= rng_state >> 7;
+ rng_state ^= rng_state << 17; return (uint32_t)(rng_state >> 32);
+}
+
+static int parsed_ok = 0, refused = 0, decoded_ok = 0, decode_refused = 0,
+ skipped_alloc = 0;
+
+static void exercise(const uint8_t *mut, uint64_t mlen, const rans_table *t,
+ uint32_t N) {
+ /* fresh allocation: blob + slack ONLY */
+ uint8_t *blob = (uint8_t *)malloc(mlen + RANS_SLACK);
+ if (!blob) return;
+ memcpy(blob, mut, mlen);
+ memset(blob + mlen, 0, RANS_SLACK);
+ rans_record rec;
+ rans_err e = rans_record_parse(blob, mlen, N, &rec);
+ if (e != RANS_OK) { refused++; free(blob); return; }
+ parsed_ok++;
+ if (rec.n_symbols > (1ull << 24) || rec.packed_bytes > (1ull << 28)) { skipped_alloc++; free(blob); return; }
+ uint8_t *out = (uint8_t *)malloc(rec.packed_bytes ? rec.packed_bytes : 1);
+ if (!out) { skipped_alloc++; free(blob); return; }
+ const rans_path arms[] = {RANS_PATH_SCALAR, RANS_PATH_SCALAR_BF,
+ RANS_PATH_NEON};
+ uint8_t *ref = NULL;
+ for (int a = 0; a < 3; a++) {
+ memset(out, 0xEE, rec.packed_bytes);
+ e = rans_record_decode_packed(&rec, t, N, arms[a], out);
+ if (e != RANS_OK) { decode_refused++; continue; }
+ decoded_ok++;
+ if (a == 0) { ref = (uint8_t *)malloc(rec.packed_bytes);
+ memcpy(ref, out, rec.packed_bytes); }
+ else if (ref && memcmp(ref, out, rec.packed_bytes) != 0) {
+ fprintf(stderr, "ARM DIVERGENCE on mutant (arm %d)\n", a);
+ exit(2);
+ }
+ }
+ /* checked per-stream decode over every stream too */
+ for (uint32_t i = 0; i < N; i++) {
+ uint64_t off0 = rec.stream_offsets[i], off1 = rec.stream_offsets[i + 1];
+ uint64_t ns = 0;
+ for (uint64_t j = i; j < rec.n_symbols; j += N) ns++;
+ if (ns > (1ull << 26)) break;
+ uint8_t *nib = (uint8_t *)malloc(ns ? ns : 1);
+ rans_decode_stream_checked(rec.payload + off0, off1 - off0, ns,
+ t->freq, t->start, t->slot_to_symbol,
+ t->scale_bits, nib);
+ free(nib);
+ }
+ free(ref); free(out); free(blob);
+}
+
+int main(void) {
+ /* base table: the 15-symbol rare-mix shape */
+ uint32_t sb = 14, M = 1u << 14;
+ uint32_t freq[16] = {0}, start[16] = {0};
+ for (int s = 1; s <= 7; s++) freq[s] = 1;
+ for (int s = 8; s <= 14; s++) freq[s] = 3;
+ freq[15] = M - 7 - 21;
+ uint64_t acc = 0;
+ for (int s = 0; s < 16; s++) { start[s] = (uint32_t)acc; acc += freq[s]; }
+ uint16_t *slot = (uint16_t *)malloc(sizeof(uint16_t) * M);
+ for (int s = 0; s < 16; s++)
+ for (uint32_t i = 0; i < freq[s]; i++) slot[start[s] + i] = (uint16_t)s;
+ rans_table t;
+ if (rans_table_init(&t, sb, freq, start, slot) != RANS_OK) return 1;
+
+ const uint32_t N = RANS_NSTREAMS;
+ const uint64_t nsyms[] = {2, 255, 256, 511, 512, 8192, 65537};
+ for (size_t bi = 0; bi < sizeof(nsyms) / sizeof(nsyms[0]); bi++) {
+ uint64_t n = nsyms[bi];
+ uint8_t *data = (uint8_t *)malloc(n);
+ for (uint64_t i = 0; i < n; i++)
+ data[i] = (uint8_t)((rnd() % 3 == 0) ? 1 + (rnd() % 14) : 15);
+ uint64_t cap = rans_record_bound(n, N) + RANS_SLACK;
+ uint8_t *base = (uint8_t *)calloc(cap, 1);
+ uint64_t blen = 0;
+ if (rans_record_encode(data, n, &t, N, base, cap, &blen) != RANS_OK)
+ return 3;
+ /* sanity: unmutated parses+decodes */
+ exercise(base, blen, &t, N);
+
+ uint8_t *mut = (uint8_t *)malloc(blen + 64);
+ for (int it = 0; it < 700; it++) {
+ uint64_t mlen = blen;
+ memcpy(mut, base, blen);
+ switch (it % 7) {
+ case 0: { /* random byte corruption, 1-8 bytes anywhere */
+ int k = 1 + (int)(rnd() % 8);
+ for (int j = 0; j < k; j++) mut[rnd() % blen] ^= (uint8_t)(1 + (rnd() & 0xFF));
+ break; }
+ case 1: { /* header field attacks */
+ uint64_t v;
+ switch (rnd() % 6) {
+ case 0: v = 0; break;
+ case 1: v = UINT64_MAX; break;
+ case 2: v = UINT64_MAX - 1; break;
+ case 3: v = (uint64_t)1 << 32; break;
+ case 4: v = n + 1; break;
+ default: v = n - 1; break;
+ }
+ if (rnd() & 1) memcpy(mut, &v, 8);
+ else memcpy(mut + 8, &v, 8);
+ /* half the time keep packed_bytes consistent with lying n */
+ if (rnd() & 1) {
+ uint64_t nn; memcpy(&nn, mut, 8);
+ uint64_t pb = (nn + 1) / 2;
+ memcpy(mut + 8, &pb, 8);
+ }
+ break; }
+ case 2: { /* offsets attacks */
+ uint32_t idx = rnd() % (N + 1);
+ uint32_t v;
+ switch (rnd() % 5) {
+ case 0: v = 0xFFFFFFFFu; break;
+ case 1: v = 0; break;
+ case 2: v = (uint32_t)(rnd()); break;
+ case 3: { uint32_t cur; memcpy(&cur, mut + 16 + 4 * (uint64_t)idx, 4);
+ v = cur + 1; break; }
+ default: { uint32_t cur; memcpy(&cur, mut + 16 + 4 * (uint64_t)idx, 4);
+ v = cur ? cur - 1 : 0; break; }
+ }
+ memcpy(mut + 16 + 4 * (uint64_t)idx, &v, 4);
+ break; }
+ case 3: /* truncate */
+ mlen = rnd() % blen;
+ break;
+ case 4: /* extend with garbage */
+ mlen = blen + 1 + rnd() % 63;
+ for (uint64_t j = blen; j < mlen; j++) mut[j] = (uint8_t)rnd();
+ break;
+ case 5: { /* swap two offsets (reversed ranges) */
+ uint32_t i1 = rnd() % (N + 1), i2 = rnd() % (N + 1);
+ uint32_t a1, a2;
+ memcpy(&a1, mut + 16 + 4 * (uint64_t)i1, 4);
+ memcpy(&a2, mut + 16 + 4 * (uint64_t)i2, 4);
+ memcpy(mut + 16 + 4 * (uint64_t)i1, &a2, 4);
+ memcpy(mut + 16 + 4 * (uint64_t)i2, &a1, 4);
+ break; }
+ default: { /* payload bit flips near stream boundaries */
+ uint32_t i1 = rnd() % N;
+ uint32_t o; memcpy(&o, base + 16 + 4 * (uint64_t)i1, 4);
+ uint64_t head = rans_record_header_bytes(N);
+ uint64_t p = head + o + (rnd() % 8);
+ if (p < blen) mut[p] ^= (uint8_t)(1u << (rnd() % 8));
+ break; }
+ }
+ if (mlen == 0) { refused++; continue; }
+ exercise(mut, mlen, &t, N);
+ }
+ free(mut); free(base); free(data);
+ }
+ printf("fuzz done: parsed_ok=%d refused=%d decoded_ok=%d decode_refused=%d skipped=%d\n",
+ parsed_ok, refused, decoded_ok, decode_refused, skipped_alloc);
+ rans_table_free(&t);
+ free(slot);
+ return 0;
+}
diff --git a/c/tests/test_rans.c b/c/tests/test_rans.c
index 11dcd81b..665865e6 100644
--- a/c/tests/test_rans.c
+++ b/c/tests/test_rans.c
@@ -615,6 +615,233 @@ static void test_path_envelope(void) {
#undef UNSET_ENV
}
+/* ---- 5. fix-round hardening regressions ---------------------------------- */
+
+/* Header-field overflow/amplification battery (u64 wrap + decompression
+ * bomb): reproduces the review round's poc_overflow.c cases and pins the
+ * non-wrapping count check + the E_OVERSIZE payload bound. */
+static void test_header_field_battery(void) {
+ const uint32_t N = RANS_NSTREAMS;
+ const uint64_t n = 8192;
+ uint8_t *d = (uint8_t *)malloc(n);
+ for (uint64_t i = 0; i < n; i++) d[i] = (uint8_t)(1 + (rnd() % 15));
+ test_table tt;
+ tt_from_hist(&tt, 14, d, n);
+ uint64_t rec_len = 0;
+ uint8_t *buf = make_record(d, n, &tt, N, &rec_len);
+ uint8_t *cp = (uint8_t *)malloc(rec_len + RANS_SLACK);
+ rans_record rec;
+ rans_err e;
+
+ struct { uint64_t n_symbols, packed_bytes; rans_err want; const char *why; }
+ cases[] = {
+ {UINT64_MAX, 0, RANS_E_COUNT_MISMATCH,
+ "u64 wrap: (n+1)/2 overflows to 0"},
+ {UINT64_MAX, (1ull << 63), RANS_E_OVERSIZE,
+ "u64 max with non-wrapping-consistent pb"},
+ {UINT64_MAX - 1, (1ull << 63) - 1, RANS_E_OVERSIZE, "u64 max-1"},
+ {1ull << 63, 1ull << 62, RANS_E_OVERSIZE, "2^63"},
+ {1ull << 40, 1ull << 39, RANS_E_OVERSIZE, "decompression bomb 2^40"},
+ {1ull << 32, 1ull << 31, RANS_E_OVERSIZE, "2^32"},
+ {(1ull << 32) + 1, (1ull << 31) + 1, RANS_E_OVERSIZE, "2^32+1"},
+ {0, 0, RANS_E_EMPTY, "zero symbols"},
+ {n + 1, (n + 1) / 2, RANS_E_COUNT_MISMATCH, "n+1, stale pb"},
+ };
+ for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) {
+ memcpy(cp, buf, rec_len);
+ memcpy(cp, &cases[c].n_symbols, 8);
+ memcpy(cp + 8, &cases[c].packed_bytes, 8);
+ e = rans_record_parse(cp, rec_len, N, &rec);
+ CHECK(e == cases[c].want, "%s: want %s got %s", cases[c].why,
+ rans_err_name(cases[c].want), rans_err_name(e));
+ }
+
+ /* minimally-framed bomb (every stream exactly its 4 flushed bytes,
+ * n_symbols = 2^40): must refuse at parse, which allocates nothing */
+ {
+ uint64_t head = rans_record_header_bytes(N);
+ uint64_t payload_len = 4ull * N;
+ uint64_t blen = head + rans_round16(payload_len);
+ uint8_t *bomb = (uint8_t *)calloc(blen + RANS_SLACK, 1);
+ uint64_t nb = 1ull << 40, pb = nb / 2;
+ memcpy(bomb, &nb, 8);
+ memcpy(bomb + 8, &pb, 8);
+ for (uint32_t i = 0; i <= N; i++) {
+ uint32_t v = 4u * i;
+ memcpy(bomb + 16 + (uint64_t)i * 4u, &v, 4);
+ }
+ for (uint64_t i = head; i < head + payload_len; i++) bomb[i] = 0x80;
+ e = rans_record_parse(bomb, blen, N, &rec);
+ CHECK(e == RANS_E_OVERSIZE, "minimal bomb: want E_OVERSIZE got %s",
+ rans_err_name(e));
+ free(bomb);
+ }
+ free(cp); free(buf); free(d);
+ tt_free(&tt);
+}
+
+/* Encoder scratch bound: bulk freq=1 input must encode fine at every
+ * scale_bits (the old n+n/2 rule failed at >= 13 β€” review poc_bound.c). */
+static void test_encoder_scratch_bound(void) {
+ for (uint32_t sb = 12; sb <= 15; sb++) {
+ uint32_t M = 1u << sb;
+ uint32_t freq[16] = {0};
+ freq[3] = 1;
+ freq[15] = M - 1;
+ test_table tt;
+ tt_build(&tt, sb, freq);
+ const uint64_t n = 8192;
+ uint8_t *d = (uint8_t *)malloc(n);
+ memset(d, 3, n); /* every symbol is the rare one */
+ uint64_t cap = rans_record_bound(n, 16) + RANS_SLACK, rl = 0;
+ uint8_t *rb = (uint8_t *)calloc(cap, 1);
+ rans_err e = rans_record_encode(d, n, &tt.t, 16, rb, cap, &rl);
+ CHECK(e == RANS_OK, "freq=1 bulk encode at sb=%u: %s", sb,
+ rans_err_name(e));
+ if (e == RANS_OK) { /* and it round-trips */
+ rans_record rec;
+ CHECK(rans_record_parse(rb, rl, 16, &rec) == RANS_OK,
+ "freq=1 sb=%u parse", sb);
+ uint8_t *out = (uint8_t *)malloc(n / 2);
+ uint8_t *ref = (uint8_t *)malloc(n / 2);
+ pack_ref(d, n, ref);
+ CHECK(rans_record_decode_packed(&rec, &tt.t, 16, RANS_PATH_SCALAR,
+ out) == RANS_OK &&
+ memcmp(out, ref, n / 2) == 0, "freq=1 sb=%u round trip", sb);
+ free(out); free(ref);
+ }
+ /* E_SCRATCH is a bound failure, never conflated with malloc failure:
+ * a too-small caller buffer names the bound, not the allocator */
+ e = rans_record_encode(d, n, &tt.t, 16, rb,
+ rans_record_header_bytes(16) + 8, &rl);
+ CHECK(e == RANS_E_SCRATCH, "small out_cap at sb=%u: want E_SCRATCH "
+ "got %s", sb, rans_err_name(e));
+ free(rb); free(d);
+ tt_free(&tt);
+ }
+}
+
+/* Uncodable input is refused by name BEFORE any encoding work: a present
+ * symbol with table freq 0, and out-of-alphabet bytes (review poc_encode.c).
+ * The stream primitive itself masks to the nibble alphabet, so garbage
+ * bytes can never index past freq[16] (byte-identical to pre-masked input). */
+static void test_uncodable_input(void) {
+ uint32_t freq[16] = {0};
+ freq[1] = 1; freq[2] = 1; freq[3] = 1;
+ freq[15] = (1u << 14) - 3; /* 4..14 have freq 0 */
+ test_table tt;
+ tt_build(&tt, 14, freq);
+ uint8_t d[64];
+ memset(d, 15, sizeof(d));
+ uint64_t cap = rans_record_bound(64, 16) + RANS_SLACK, rl = 0;
+ uint8_t *rb = (uint8_t *)calloc(cap, 1);
+ rans_err e;
+
+ d[7] = 9; /* freq[9] == 0 */
+ e = rans_record_encode(d, 64, &tt.t, 16, rb, cap, &rl);
+ CHECK(e == RANS_E_SYMBOL_UNCODABLE, "freq=0 symbol: want "
+ "E_SYMBOL_UNCODABLE got %s", rans_err_name(e));
+
+ d[7] = 200; /* out of the nibble alphabet */
+ e = rans_record_encode(d, 64, &tt.t, 16, rb, cap, &rl);
+ CHECK(e == RANS_E_SYMBOL_UNCODABLE, "value 200: want E_SYMBOL_UNCODABLE "
+ "got %s", rans_err_name(e));
+
+ /* stream primitive: masked, so 200 encodes as 200&15 == 8, byte-exact */
+ uint32_t f2[16];
+ for (int s = 0; s < 16; s++) f2[s] = (1u << 14) / 16;
+ test_table t2;
+ tt_build(&t2, 14, f2);
+ uint8_t da[64], db[64], ba[512], bb[512];
+ for (int i = 0; i < 64; i++) da[i] = db[i] = (uint8_t)(rnd() & 15);
+ da[3] = 200; db[3] = 200 & 15;
+ size_t oa = rans_encode_stream(da, 64, t2.t.freq, t2.t.start,
+ t2.t.scale_bits, ba, sizeof(ba));
+ size_t ob = rans_encode_stream(db, 64, t2.t.freq, t2.t.start,
+ t2.t.scale_bits, bb, sizeof(bb));
+ CHECK(oa == ob && oa != (size_t)-1 &&
+ memcmp(ba + oa, bb + ob, sizeof(ba) - oa) == 0,
+ "stream mask: 200 must encode as 200&15, byte-identical");
+ free(rb);
+ tt_free(&tt); tt_free(&t2);
+}
+
+/* n_streams validation + record-buffer alignment refusals. */
+static void test_nstreams_and_alignment(void) {
+ const uint64_t n = 512;
+ uint8_t *d = (uint8_t *)malloc(n);
+ for (uint64_t i = 0; i < n; i++) d[i] = (uint8_t)(1 + (rnd() % 15));
+ test_table tt;
+ tt_from_hist(&tt, 14, d, n);
+ uint64_t rec_len = 0;
+ uint8_t *buf = make_record(d, n, &tt, RANS_NSTREAMS, &rec_len);
+ rans_record rec;
+ rans_err e;
+
+ CHECK(rans_record_bound(n, 0) == 0, "bound with n_streams=0 returns 0");
+ uint64_t rl = 0;
+ e = rans_record_encode(d, n, &tt.t, 0, buf, rec_len, &rl);
+ CHECK(e == RANS_E_NSTREAMS, "encode n_streams=0: %s", rans_err_name(e));
+ e = rans_record_parse(buf, rec_len, 0, &rec);
+ CHECK(e == RANS_E_NSTREAMS, "parse n_streams=0: %s", rans_err_name(e));
+ CHECK(rans_record_parse(buf, rec_len, RANS_NSTREAMS, &rec) == RANS_OK,
+ "control parse");
+ uint8_t out[256];
+ e = rans_record_decode_packed(&rec, &tt.t, 0, RANS_PATH_SCALAR, out);
+ CHECK(e == RANS_E_NSTREAMS, "decode n_streams=0: %s", rans_err_name(e));
+
+ /* a misaligned record buffer is refused by name, not cast through */
+ uint8_t *mis = (uint8_t *)malloc(rec_len + RANS_SLACK + 1);
+ memcpy(mis + 1, buf, rec_len);
+ e = rans_record_parse(mis + 1, rec_len, RANS_NSTREAMS, &rec);
+ CHECK(e == RANS_E_UNALIGNED, "misaligned blob: %s", rans_err_name(e));
+ free(mis); free(buf); free(d);
+ tt_free(&tt);
+}
+
+/* Band coverage as a checked property of the suite's own rare-symbol
+ * fixture (the runtime selftest asserts the same on its fixture): the
+ * multi-byte renorm band the sabotage exercise proved load-bearing must
+ * actually be visited, or the identity sweep is running blind. */
+static void test_band_coverage(void) {
+ const uint64_t n = 32768;
+ uint8_t *d = (uint8_t *)malloc(n);
+ uint32_t rare[16] = {0};
+ for (int s = 1; s <= 7; s++) rare[s] = 1;
+ for (int s = 8; s <= 14; s++) rare[s] = 3;
+ rare[15] = (1u << 14) - 7 - 21;
+ test_table tt;
+ tt_build(&tt, 14, rare);
+ for (uint64_t i = 0; i < n; i++)
+ d[i] = (uint8_t)((rnd() % 3 == 0) ? 1 + (rnd() % 14) : 15);
+ uint64_t rec_len = 0;
+ uint8_t *buf = make_record(d, n, &tt, RANS_NSTREAMS, &rec_len);
+ rans_record rec;
+ CHECK(rans_record_parse(buf, rec_len, RANS_NSTREAMS, &rec) == RANS_OK,
+ "band fixture parse");
+ uint64_t kb2 = 0, steps = 0;
+ for (uint32_t i = 0; i < RANS_NSTREAMS; i++) {
+ const uint8_t *ptr = rec.payload + rec.stream_offsets[i];
+ const uint8_t *end = rec.payload + rec.stream_offsets[i + 1];
+ uint32_t x = 0;
+ for (int b = 0; b < 4; b++) x = (x << 8) | *ptr++;
+ for (uint64_t j = i; j < rec.n_symbols; j += RANS_NSTREAMS) {
+ uint32_t slot = x & (tt.t.M - 1u);
+ uint32_t s = tt.slot[slot];
+ x = tt.t.freq[s] * (x >> tt.t.scale_bits) + slot - tt.t.start[s];
+ uint32_t kb = 0;
+ while (x < RANS_L) { if (ptr >= end) break; x = (x << 8) | *ptr++; kb++; }
+ if (kb >= 2) kb2++;
+ steps++;
+ }
+ }
+ CHECK(kb2 > steps / 100, "multi-byte renorm band: %llu of %llu steps "
+ "(need > 1%%) β€” the rare-symbol fixture lost its coverage",
+ (unsigned long long)kb2, (unsigned long long)steps);
+ free(buf); free(d);
+ tt_free(&tt);
+}
+
int main(void) {
test_stream_property_suite();
test_stream_checked_refusals();
@@ -622,6 +849,11 @@ int main(void) {
test_record_refusals();
test_arm_identity_suite();
test_path_envelope();
+ test_header_field_battery();
+ test_encoder_scratch_bound();
+ test_uncodable_input();
+ test_nstreams_and_alignment();
+ test_band_coverage();
if (failures) {
printf("test_rans: FAIL (%d)\n", failures);
return 1;
diff --git a/c/tests/test_rans_repack.py b/c/tests/test_rans_repack.py
index 9ea8a268..544e79c5 100644
--- a/c/tests/test_rans_repack.py
+++ b/c/tests/test_rans_repack.py
@@ -44,7 +44,8 @@ def run_tool(script, *args, env_extra=None):
@unittest.skipIf(rf is None, "numpy not available (offline-tooling test)")
class TestRansRepack(unittest.TestCase):
- O, I = 8, 128 # per-row ratio I/2 = 64 (safely above the g64 signature)
+ O, I = 8, 256 # per-row ratio I/2 = 128, above every grouped
+ # signature (32 = g64, 64 = g128/ambiguous)
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
@@ -107,15 +108,32 @@ class TestRansRepack(unittest.TestCase):
name + ".qs")
self.assertEqual(qblob, qs, name + ".qs")
- # determinism: an independent second run is byte-identical
+ # determinism: an independent second run is byte-identical, manifest
+ # (the completion marker) included
out2 = self.repack(self.root / "out2")
self.assertEqual(out1[0].read_bytes(), out2[0].read_bytes())
+ self.assertEqual(
+ (out1[0].parent / "repack-manifest.json").read_bytes(),
+ (out2[0].parent / "repack-manifest.json").read_bytes())
# implementation identity: a forced pure-Python run emits the SAME
- # bytes as the (lib-backed, when built) default run
+ # bytes as the C-bridge run. VACUOUS unless the bridge is actually
+ # loaded, so that is asserted β€” `make test-python` builds the bridge;
+ # a direct unittest invocation without it skips LOUDLY instead of
+ # green-lighting a pin it never checked.
+ if rf.LIB is None:
+ self.skipTest("tools/librans_c not built (run `make rans`): "
+ "the C-vs-Python byte-identity pin was NOT checked")
out3 = self.repack(self.root / "out3", RANS_FORCE_PYTHON="1")
self.assertEqual(out1[0].read_bytes(), out3[0].read_bytes())
+ # bridge-presence: the default run must actually have used the C
+ # codec (not fallen back to Python with the lib sitting there)
+ r = run_tool("repack_rans.py", "--indir", str(self.indir),
+ "--outdir", str(self.root / "outdry"), "--dry-run")
+ self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr)
+ self.assertIn("codec: C (librans_c)", r.stdout)
+
# the validator trusts it
r = run_tool("rans_verify.py", str(out1[0]))
self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr)
@@ -137,9 +155,12 @@ class TestRansRepack(unittest.TestCase):
meta = meta_mutator(meta)
rf.write_shard(str(shard), tensors, meta)
+ _corrupt_seq = 0
+
def _corrupt_reshard_and_expect(self, expected_code, meta_mutator=None,
tensor_mutator=None):
- outdir = self.root / f"corrupt_{expected_code}"
+ TestRansRepack._corrupt_seq += 1
+ outdir = self.root / f"corrupt_{TestRansRepack._corrupt_seq}_{expected_code}"
shard = self.repack(outdir)[0]
self._reshard(shard, meta_mutator, tensor_mutator)
r = run_tool("rans_verify.py", str(shard))
@@ -177,6 +198,15 @@ class TestRansRepack(unittest.TestCase):
blob[16 + (rf.N_STREAMS + 1) * 4] = 0x77 # header pad byte
return blob
+ def bomb_header(name, blob):
+ if name == target:
+ # n_symbols = 2^40 with a consistent packed_bytes: the
+ # "oversize fields" class the spec names β€” must refuse as
+ # E_OVERSIZE, never an allocator traceback
+ blob[0:8] = (1 << 40).to_bytes(8, "little")
+ blob[8:16] = (1 << 39).to_bytes(8, "little")
+ return blob
+
def drop_stamp(meta):
del meta[rf.METADATA_KEY]
return meta
@@ -206,6 +236,7 @@ class TestRansRepack(unittest.TestCase):
# payload corruption trips a stream invariant or the re-encode pin;
# both are refusals β€” assert on the shared prefix
self._corrupt_reshard_and_expect("E_", tensor_mutator=flip_payload)
+ self._corrupt_reshard_and_expect("E_OVERSIZE", tensor_mutator=bomb_header)
self._corrupt_reshard_and_expect("E_COUNT_MISMATCH",
tensor_mutator=flip_count)
self._corrupt_reshard_and_expect("E_", tensor_mutator=break_offsets)
@@ -235,6 +266,46 @@ class TestRansRepack(unittest.TestCase):
self.assertEqual(r.returncode, 1)
self.assertIn("E_GEOMETRY_G64", r.stderr)
+ # gs=128 signature (upstream grouped-int4's actual group size):
+ # ratio exactly 64, byte-indistinguishable from per-row I=128 ->
+ # its own named refusal, never silently accepted as per-row
+ g128dir = self.root / "g128"
+ g128dir.mkdir()
+ rows128 = wb // 64
+ rf.write_shard(str(g128dir / "out-00000.safetensors"), [
+ ("model.layers.0.mlp.experts.0.gate_proj.weight", "U8", [wb],
+ bytes(wb)),
+ ("model.layers.0.mlp.experts.0.gate_proj.weight.qs", "F32",
+ [rows128], bytes(rows128 * 4)),
+ ], {})
+ r = run_tool("repack_rans.py", "--indir", str(g128dir),
+ "--outdir", str(self.root / "g128out"))
+ self.assertEqual(r.returncode, 1)
+ self.assertIn("E_GEOMETRY_AMBIGUOUS", r.stderr)
+
+ # 2-D weight shape: per-row geometry is verified POSITIVELY from the
+ # shape fields β€” accepted when qs rows == O, refused by name when not
+ rng = np.random.default_rng(3)
+ O, rb2 = 8, 64
+ nib = rng.choice(np.arange(1, 16, dtype=np.uint8), size=O * rb2 * 2)
+ raw = rf.pack_nibbles(nib)
+ for rows2d, expect_ok in ((O, True), (O * 2, False)):
+ d2 = self.root / f"twod_{rows2d}"
+ d2.mkdir()
+ rf.write_shard(str(d2 / "out-00000.safetensors"), [
+ ("model.layers.0.mlp.experts.0.gate_proj.weight", "U8",
+ [O, rb2], raw),
+ ("model.layers.0.mlp.experts.0.gate_proj.weight.qs", "F32",
+ [rows2d], bytes(rows2d * 4)),
+ ], {})
+ r = run_tool("repack_rans.py", "--indir", str(d2),
+ "--outdir", str(self.root / f"twod_out_{rows2d}"))
+ if expect_ok:
+ self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr)
+ else:
+ self.assertEqual(r.returncode, 1)
+ self.assertIn("E_GEOMETRY_NOT_PER_ROW", r.stderr)
+
# missing .qs sidecar -> named refusal
nsdir = self.root / "noscales"
nsdir.mkdir()
@@ -258,6 +329,162 @@ class TestRansRepack(unittest.TestCase):
self.assertEqual(r.returncode, 1)
self.assertIn("refusing to emit an empty container", r.stderr)
+ def test_parity_c_python_refusal_classes(self):
+ """Same bytes into the C parser (via the bridge) and the Python
+ parser must produce the same named class β€” parity is a tested
+ property, not a documented intention. Covers the u64-wrap and
+ decompression-bomb header battery."""
+ if rf.LIB is None:
+ self.skipTest("tools/librans_c not built (run `make rans`): "
+ "C/Python refusal parity was NOT checked")
+ import ctypes
+
+ def c_parse(blob):
+ buf = np.zeros(len(blob) + rf.SLACK, dtype=np.uint8)
+ buf[:len(blob)] = np.frombuffer(blob, dtype=np.uint8)
+ ns = ctypes.c_uint64()
+ pb = ctypes.c_uint64()
+ rc = rf.LIB.rc_record_parse(rf._p8(buf), len(blob), rf.N_STREAMS,
+ ctypes.byref(ns), ctypes.byref(pb))
+ return "OK" if rc == 0 else rf.LIB.rc_err_name(rc).decode()
+
+ def py_parse(blob):
+ try:
+ rf.parse_record(blob)
+ return "OK"
+ except rf.RansRefusal as exc:
+ return exc.code
+
+ rng = np.random.default_rng(11)
+ n = 4096
+ nib = rng.choice(np.arange(1, 16, dtype=np.uint8), size=n)
+ hist = np.bincount(nib, minlength=16)
+ freq = rf.quantize_freq(hist)
+ start, slot = rf.build_table(freq)
+ base = bytearray(rf.build_record(nib, freq, start, slot))
+ u64max = (1 << 64) - 1
+
+ def hdr(ns, pb):
+ b = bytearray(base)
+ b[0:8] = ns.to_bytes(8, "little")
+ b[8:16] = pb.to_bytes(8, "little")
+ return bytes(b)
+
+ cases = [bytes(base)] # the valid control
+ for ns, pb in ((u64max, 0), (u64max, 1 << 63), (0, 0),
+ (1 << 40, 1 << 39), (1 << 32, 1 << 31),
+ (n + 1, (n + 1) // 2), (n, n)):
+ cases.append(hdr(ns, pb))
+ cases.append(bytes(base[:64])) # truncated header
+ cases.append(bytes(base[:len(base) - 16])) # truncated payload
+ cases.append(bytes(base) + b"\x00" * 16) # trailing pad-shaped
+ cases.append(bytes(base) + b"\x77" * 16) # trailing garbage
+ mono = bytearray(base)
+ mono[16 + 8 * 4] ^= 0x80 # offsets landmine
+ cases.append(bytes(mono))
+ dirty = bytearray(base)
+ dirty[16 + (rf.N_STREAMS + 1) * 4] = 0x55 # header pad byte
+ cases.append(bytes(dirty))
+
+ for i, blob in enumerate(cases):
+ c_cls, py_cls = c_parse(blob), py_parse(blob)
+ self.assertEqual(c_cls, py_cls,
+ msg=f"case {i}: C={c_cls} Python={py_cls}")
+ self.assertEqual(c_parse(cases[0]), "OK") # control really is OK
+
+ def test_verifier_never_crashes(self):
+ """Hostile safetensors headers (the review round's traceback
+ classes): zero tracebacks, nonzero exit, a named REFUSE line each."""
+ name = "model.layers.0.mlp.experts.0.gate_proj.weight"
+ ok_entry = {"dtype": "U8", "shape": [4], "data_offsets": [0, 4]}
+ stamp = json.dumps({name: rf.FORMAT_NAME})
+ cases = {
+ "meta_not_string": {"__metadata__": {rf.METADATA_KEY: 5},
+ name: dict(ok_entry)},
+ "header_array": [1, 2, 3],
+ "no_data_offsets": {"__metadata__": {rf.METADATA_KEY: stamp,
+ rf.TABLE_KEY: "{}"},
+ name: {"dtype": "U8", "shape": [4]}},
+ "entry_not_dict": {"__metadata__": {rf.METADATA_KEY: stamp,
+ rf.TABLE_KEY: "{}"},
+ name: "hello"},
+ "fmt_value_list": {"__metadata__":
+ {rf.METADATA_KEY: json.dumps({name: ["a"]})},
+ name: dict(ok_entry)},
+ "reversed_offsets": {"__metadata__": {rf.METADATA_KEY: stamp,
+ rf.TABLE_KEY: "{}"},
+ name: {"dtype": "U8", "shape": [4],
+ "data_offsets": [8, 2]}},
+ "offsets_past_eof": {"__metadata__": {rf.METADATA_KEY: stamp,
+ rf.TABLE_KEY: "{}"},
+ name: {"dtype": "U8", "shape": [4],
+ "data_offsets": [0, 4096]}},
+ "offsets_not_pair": {"__metadata__": {rf.METADATA_KEY: stamp,
+ rf.TABLE_KEY: "{}"},
+ name: {"dtype": "U8", "shape": [4],
+ "data_offsets": [1, 2, 3]}},
+ }
+ hostile = self.root / "hostile"
+ hostile.mkdir()
+ for label, hdr in cases.items():
+ p = hostile / f"h_{label}.safetensors"
+ hj = json.dumps(hdr, separators=(",", ":")).encode()
+ with open(p, "wb") as f:
+ f.write(len(hj).to_bytes(8, "little"))
+ f.write(hj)
+ f.write(b"\x00" * 32)
+ r = run_tool("rans_verify.py", str(p))
+ self.assertNotIn("Traceback", r.stderr,
+ msg=f"{label}: crashed\n{r.stderr}")
+ self.assertEqual(r.returncode, 1, msg=f"{label}: {r.stdout}")
+ self.assertIn("REFUSE", r.stdout, msg=f"{label}: no REFUSE line")
+
+ def test_partial_output_refusal(self):
+ """An outdir already holding repack output is refused (crash
+ evidence: an interrupted set must not be papered over); --force
+ redoes it; a completed run leaves repack-manifest.json."""
+ out = self.root / "partial"
+ shards = self.repack(out)
+ manifest = out / "repack-manifest.json"
+ self.assertTrue(manifest.exists())
+ man = json.loads(manifest.read_text())
+ self.assertEqual(man["n_shards"], len(shards))
+
+ # rerun over completed output: refused
+ r = run_tool("repack_rans.py", "--indir", str(self.indir),
+ "--outdir", str(out))
+ self.assertEqual(r.returncode, 1)
+ self.assertIn("refusing to overwrite", r.stderr)
+
+ # simulate a crashed run: shards present, no completion manifest
+ manifest.unlink()
+ r = run_tool("repack_rans.py", "--indir", str(self.indir),
+ "--outdir", str(out))
+ self.assertEqual(r.returncode, 1)
+ self.assertIn("NO manifest", r.stderr)
+
+ # --force redoes from scratch, byte-identical to a fresh run
+ r = run_tool("repack_rans.py", "--indir", str(self.indir),
+ "--outdir", str(out), "--force")
+ self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr)
+ self.assertTrue(manifest.exists())
+ fresh = self.repack(self.root / "partial_fresh")
+ self.assertEqual(shards[0].read_bytes(), fresh[0].read_bytes())
+
+ def test_quantize_freq_ties_and_termination(self):
+ """Exact fractional-remainder ties must break identically everywhere
+ (stable argsort -> lowest symbol index wins), and an impossible table
+ size must raise instead of looping forever."""
+ hist = np.zeros(16, dtype=np.int64)
+ hist[1] = hist[2] = hist[3] = 1 # remainders tie exactly at 1/3
+ freq = rf.quantize_freq(hist)
+ self.assertEqual(freq.tolist(),
+ [0, 5462, 5461, 5461, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0])
+ self.assertEqual(int(freq.sum()), rf.M)
+ with self.assertRaises(ValueError):
+ rf.quantize_freq(np.ones(16, dtype=np.int64), m=8)
+
if __name__ == "__main__":
unittest.main()
diff --git a/c/tools/rans_ctypes.c b/c/tools/rans_ctypes.c
index e316ad45..ce61e685 100644
--- a/c/tools/rans_ctypes.c
+++ b/c/tools/rans_ctypes.c
@@ -15,6 +15,14 @@
#define RC_EXPORT
#endif
+/* This bridge serves the int4-rans256-g0 TOOLS, so unlike the format-generic
+ * header entry points it can and does pin n_streams to the format's 256 β€”
+ * a zero or wrong-for-format value is refused by name before any division
+ * or decode dispatch (the header entries additionally refuse zero). */
+static int rc__nstreams_ok(uint32_t n_streams) {
+ return n_streams == RANS_NSTREAMS;
+}
+
/* Encode n nibbles into one chunk record. Returns the record length, or the
* negated rans_err on failure. out_cap should come from rc_record_bound. */
RC_EXPORT int64_t rc_record_encode(const uint8_t *nibbles, uint64_t n,
@@ -22,6 +30,7 @@ RC_EXPORT int64_t rc_record_encode(const uint8_t *nibbles, uint64_t n,
const uint16_t *slot_to_symbol,
uint32_t scale_bits, uint32_t n_streams,
uint8_t *out, uint64_t out_cap) {
+ if (!rc__nstreams_ok(n_streams)) return -(int64_t)RANS_E_NSTREAMS;
rans_table t;
rans_err e = rans_table_init(&t, scale_bits, freq, start, slot_to_symbol);
if (e != RANS_OK) return -(int64_t)e;
@@ -33,6 +42,7 @@ RC_EXPORT int64_t rc_record_encode(const uint8_t *nibbles, uint64_t n,
}
RC_EXPORT uint64_t rc_record_bound(uint64_t n_symbols, uint32_t n_streams) {
+ if (!rc__nstreams_ok(n_streams)) return 0; /* callers treat 0 as refusal */
return rans_record_bound(n_symbols, n_streams);
}
@@ -41,6 +51,7 @@ RC_EXPORT uint64_t rc_record_bound(uint64_t n_symbols, uint32_t n_streams) {
RC_EXPORT int32_t rc_record_parse(const uint8_t *blob, uint64_t blob_len,
uint32_t n_streams,
uint64_t *n_symbols, uint64_t *packed_bytes) {
+ if (!rc__nstreams_ok(n_streams)) return (int32_t)RANS_E_NSTREAMS;
rans_record rec;
rans_err e = rans_record_parse(blob, blob_len, n_streams, &rec);
if (e == RANS_OK) {
@@ -61,6 +72,7 @@ RC_EXPORT int32_t rc_record_decode(const uint8_t *blob, uint64_t blob_len,
const uint16_t *slot_to_symbol,
uint32_t scale_bits, int32_t path,
uint8_t *out_packed) {
+ if (!rc__nstreams_ok(n_streams)) return (int32_t)RANS_E_NSTREAMS;
rans_record rec;
rans_err e = rans_record_parse(blob, blob_len, n_streams, &rec);
if (e != RANS_OK) return (int32_t)e;
diff --git a/c/tools/rans_format.py b/c/tools/rans_format.py
index a90314f5..4c6e3614 100644
--- a/c/tools/rans_format.py
+++ b/c/tools/rans_format.py
@@ -64,10 +64,25 @@ class RansRefusal(Exception):
def quantize_freq(hist, m=M):
"""hist: length-16 array of raw counts. Returns freq[16] uint32 summing to
m; largest-remainder quantization; min 1 count for any nonzero-count
- symbol (so it stays encodable); absent symbols get exactly 0."""
+ symbol (so it stays encodable); absent symbols get exactly 0.
+
+ Determinism: both argsorts use kind="stable" so exact fractional-remainder
+ (or frequency) ties break by symbol index identically on every host and
+ numpy version β€” contract 4's cross-host byte-identity depends on it.
+
+ Termination: requires m >= the number of present symbols (raises
+ ValueError otherwise β€” the min-1 constraint would be unsatisfiable).
+ Given that, the deficit<0 loop provably terminates: while deficit < 0,
+ sum(freq) = m - deficit > m >= n_present, so some entry exceeds 1 and
+ every full pass over `order` decrements at least one entry."""
hist = np.asarray(hist, dtype=np.float64)
total = hist.sum()
assert total > 0
+ n_present = int(np.count_nonzero(hist))
+ if m < n_present:
+ raise ValueError(
+ f"quantize_freq: m={m} < {n_present} present symbols β€” every "
+ f"present symbol needs freq >= 1, so this table size is impossible")
raw = hist / total * m
freq = np.floor(raw).astype(np.int64)
nonzero = hist > 0
@@ -76,7 +91,7 @@ def quantize_freq(hist, m=M):
if deficit > 0:
remainder = raw - np.floor(raw)
remainder[~nonzero] = -1 # never give slots to symbols absent from data
- order = np.argsort(-remainder)
+ order = np.argsort(-remainder, kind="stable")
i = 0
while deficit > 0 and i < len(order):
idx = order[i]
@@ -85,7 +100,7 @@ def quantize_freq(hist, m=M):
deficit -= 1
i += 1
elif deficit < 0:
- order = np.argsort(-freq)
+ order = np.argsort(-freq, kind="stable")
i = 0
while deficit < 0:
idx = order[i % len(order)]
@@ -317,8 +332,18 @@ def build_record(nibbles, freq, start, slot_to_symbol, scale_bits=SCALE_BITS,
"""nibbles -> one chunk-record bytes object. Deterministic."""
nib = np.ascontiguousarray(nibbles, dtype=np.uint8)
n = nib.size
+ if n_streams != N_STREAMS:
+ raise RansRefusal("E_NSTREAMS",
+ f"n_streams={n_streams} (this format fixes {N_STREAMS})")
if n == 0:
raise RansRefusal("E_EMPTY", "refusing to encode an empty tensor")
+ # pre-scan parity with c/rans.h: refuse uncodable input by name before
+ # any encoding work (values > 15, or symbols the table gives freq 0)
+ freq_a = np.asarray(freq, dtype=np.uint32)
+ if int(nib.max(initial=0)) > 15 or (freq_a[nib] == 0).any():
+ raise RansRefusal("E_SYMBOL_UNCODABLE",
+ "input contains a value > 15 or a symbol whose "
+ "table frequency is zero")
if LIB is not None:
cap = int(LIB.rc_record_bound(n, n_streams)) + SLACK
out = np.zeros(cap, dtype=np.uint8)
@@ -345,13 +370,19 @@ def build_record(nibbles, freq, start, slot_to_symbol, scale_bits=SCALE_BITS,
def parse_record(blob, n_streams=N_STREAMS):
"""Validate framing; returns dict(n_symbols, packed_bytes, offsets,
- payload). Raises RansRefusal with the same named classes as c/rans.h."""
+ payload). Raises RansRefusal with the same named classes as c/rans.h β€”
+ that parity is a TESTED property (the e2e suite feeds identical bytes to
+ both implementations and asserts identical classes)."""
+ if n_streams == 0:
+ raise RansRefusal("E_NSTREAMS", "n_streams == 0")
head = record_header_bytes(n_streams)
if len(blob) < head:
raise RansRefusal("E_TRUNCATED", f"{len(blob)} bytes < {head}-byte header")
n_symbols, packed_bytes = struct.unpack_from("<QQ", blob, 0)
if n_symbols == 0:
raise RansRefusal("E_EMPTY", "n_symbols == 0")
+ # Python ints do not wrap, so ceil(n/2) here already agrees with the C
+ # side's non-wrapping n/2 + (n&1) for every uint64 value incl. UINT64_MAX
if packed_bytes != (n_symbols + 1) // 2:
raise RansRefusal("E_COUNT_MISMATCH",
f"packed_bytes={packed_bytes} != ceil({n_symbols}/2)")
@@ -372,6 +403,14 @@ def parse_record(blob, n_streams=N_STREAMS):
raise RansRefusal("E_LENGTH_MISMATCH",
f"blob is {len(blob)} bytes, framing derives "
f"{head + _round16(payload_len)}")
+ # amplification bound, identical to c/rans.h: no table this format admits
+ # can encode more than payload_len*8*M_max symbols into this payload β€”
+ # refuse the decompression bomb before anything sizes buffers from
+ # n_symbols
+ if n_symbols > payload_len * 8 * (1 << 15):
+ raise RansRefusal("E_OVERSIZE",
+ f"n_symbols={n_symbols} impossibly large for a "
+ f"{payload_len}-byte payload")
raw_head_end = 16 + (n_streams + 1) * 4
if any(blob[raw_head_end:head]) or any(blob[head + payload_len:]):
raise RansRefusal("E_PAD_NONZERO", "derived padding bytes are not zero")
@@ -390,10 +429,18 @@ def decode_record(blob, table, n_streams=N_STREAMS, checked=True):
slot = np.ascontiguousarray(table["slot_to_symbol"], dtype=np.uint16)
sb = table["scale_bits"]
n = rec["n_symbols"]
+ # parse_record bounded n_symbols against the payload (E_OVERSIZE); a
+ # record that legitimately passes can still be too big for THIS machine β€”
+ # that is a named refusal, never an allocator traceback
+ try:
+ nib = np.empty(n, dtype=np.uint8)
+ out = np.empty(rec["packed_bytes"], dtype=np.uint8)
+ except MemoryError:
+ raise RansRefusal("E_NOMEM",
+ f"cannot allocate buffers for n_symbols={n}")
if LIB is not None and not checked:
buf = np.zeros(len(blob) + SLACK, dtype=np.uint8) # SLACK contract
buf[:len(blob)] = np.frombuffer(blob, dtype=np.uint8)
- out = np.empty(rec["packed_bytes"], dtype=np.uint8)
rc = LIB.rc_record_decode(_p8(buf), len(blob), n_streams, _p32(freq),
_p32(start), _p16(slot), sb, -1, _p8(out))
if rc != 0:
@@ -402,7 +449,6 @@ def decode_record(blob, table, n_streams=N_STREAMS, checked=True):
# checked per-stream decode (verification path), C-accelerated per stream
payload = rec["payload"]
offsets = rec["offsets"]
- nib = np.empty(n, dtype=np.uint8)
for i in range(n_streams):
sub = payload[int(offsets[i]):int(offsets[i + 1])]
n_i = len(range(i, n, n_streams))
@@ -431,23 +477,58 @@ def decode_record(blob, table, n_streams=N_STREAMS, checked=True):
# external safetensors dependency, same spirit as doctor.py's own parser
# ---------------------------------------------------------------------------
def read_header(path):
+ """Parse AND validate a shard header. An untrusted container reaches the
+ tools through this function, so the whole header shape is pinned here β€”
+ every malformation is a named E_SHARD_MALFORMED refusal, never a
+ downstream KeyError/TypeError traceback: the header must be a JSON
+ object; every tensor entry an object with a string dtype, a list shape,
+ and data_offsets = [b, e] with 0 <= b <= e <= data size; __metadata__
+ (when present) a string-to-string object."""
+ fsize = os.path.getsize(path)
with open(path, "rb") as f:
raw = f.read(8)
if len(raw) != 8:
raise RansRefusal("E_SHARD_MALFORMED", f"{path}: no header length")
hlen = struct.unpack("<Q", raw)[0]
- if hlen > 512 * 1024 * 1024:
+ if hlen > 512 * 1024 * 1024 or 8 + hlen > fsize:
raise RansRefusal("E_SHARD_MALFORMED", f"{path}: absurd header ({hlen} B)")
try:
header = json.loads(f.read(hlen))
except ValueError as exc:
raise RansRefusal("E_SHARD_MALFORMED", f"{path}: header not JSON: {exc}")
- return header, 8 + hlen
+ if not isinstance(header, dict):
+ raise RansRefusal("E_SHARD_MALFORMED",
+ f"{path}: header is {type(header).__name__}, not an object")
+ data_start = 8 + hlen
+ data_size = fsize - data_start
+ for name, info in header.items():
+ if name == "__metadata__":
+ if not isinstance(info, dict) or \
+ not all(isinstance(k, str) and isinstance(v, str)
+ for k, v in info.items()):
+ raise RansRefusal("E_SHARD_MALFORMED",
+ f"{path}: __metadata__ is not a str->str object")
+ continue
+ if not isinstance(info, dict):
+ raise RansRefusal("E_SHARD_MALFORMED",
+ f"{path}:{name}: tensor entry is not an object")
+ if not isinstance(info.get("dtype"), str) or \
+ not isinstance(info.get("shape"), list):
+ raise RansRefusal("E_SHARD_MALFORMED",
+ f"{path}:{name}: missing/malformed dtype or shape")
+ off = info.get("data_offsets")
+ if (not isinstance(off, list) or len(off) != 2 or
+ not all(isinstance(x, int) and not isinstance(x, bool) for x in off) or
+ not (0 <= off[0] <= off[1] <= data_size)):
+ raise RansRefusal("E_SHARD_MALFORMED",
+ f"{path}:{name}: data_offsets {off!r} invalid for "
+ f"{data_size} data bytes")
+ return header, data_start
def read_tensor_bytes(path, header, data_start, name):
info = header[name]
- b, e = info["data_offsets"]
+ b, e = info["data_offsets"] # validated by read_header
with open(path, "rb") as f:
f.seek(data_start + b)
raw = f.read(e - b)
diff --git a/c/tools/rans_verify.py b/c/tools/rans_verify.py
index 234f8f92..efe692dd 100644
--- a/c/tools/rans_verify.py
+++ b/c/tools/rans_verify.py
@@ -4,6 +4,15 @@ boundary. Accepts a spec-conformant shard; produces a NAMED refusal for every
corruption class; never crashes, never silently accepts.
WHAT IS CHECKED, per shard:
+ container layer:
+ E_SHARD_MALFORMED unreadable/absurd header; header not a JSON
+ object; tensor entries not objects; dtype/shape
+ missing; data_offsets absent, non-[b,e],
+ reversed, or past the end of the file;
+ __metadata__ not a str->str object
+ E_INTERNAL anything unexpected β€” a catch-all that converts
+ any surprise into a named refusal so "never
+ crashes" is a property, not a hope
stamp layer (the stamp is MANDATORY for this format β€” entropy-coded sizes
are data-dependent, so the stamp is the ONLY signal a U8 tensor is
entropy-coded at all):
@@ -22,9 +31,11 @@ WHAT IS CHECKED, per shard:
E_TABLE_START start[] not the prefix sum of freq[]
E_TABLE_SLOT slot_to_symbol inconsistent with freq[]/start[]
E_TABLE_CRC slot-table bytes do not match the stamped crc32
- record layer (per stamped tensor; same names as c/rans.h's rans_err):
- E_TRUNCATED E_EMPTY E_COUNT_MISMATCH E_OFFSET_FIRST E_OFFSETS_MONOTONIC
- E_STREAM_SHORT E_LENGTH_MISMATCH E_PAD_NONZERO
+ record layer (per stamped tensor; same names as c/rans.h's rans_err, and
+ the same-bytes/same-class parity with the C parser is a tested property):
+ E_TRUNCATED E_EMPTY E_COUNT_MISMATCH E_OVERSIZE E_OFFSET_FIRST
+ E_OFFSETS_MONOTONIC E_STREAM_SHORT E_LENGTH_MISMATCH E_PAD_NONZERO
+ E_NOMEM (a record too large for this machine refuses by name)
payload layer (per stream; a genuine encoder output ALWAYS satisfies
these, so corruption is refused without any ground-truth bytes):
E_STREAM_STATE_RANGE initial state outside the encoder's invariant
@@ -61,6 +72,8 @@ def refuse(failures, shard, tensor, code, detail):
def verify_shard(path, failures, max_tensors=None):
+ """One shard. Wrapped by main()'s catch-all so no input can escape as a
+ traceback; rf.read_header pins the whole header shape by name first."""
try:
header, data_start = rf.read_header(path)
except rf.RansRefusal as exc:
@@ -111,37 +124,31 @@ def verify_shard(path, failures, max_tensors=None):
f"--max-tensors {max_tensors} of {len(ours)} stamped tensors")
break
checked += 1
- if name not in header:
- refuse(failures, path, name, "E_STAMP_DANGLING",
- "stamped but not present in this shard")
- continue
- info = header[name]
- if info["dtype"] != "U8":
- refuse(failures, path, name, "E_STAMP_DTYPE",
- f"dtype {info['dtype']} != U8")
- continue
try:
+ if name not in header:
+ raise rf.RansRefusal("E_STAMP_DANGLING",
+ "stamped but not present in this shard")
+ info = header[name]
+ if info["dtype"] != "U8":
+ raise rf.RansRefusal("E_STAMP_DTYPE",
+ f"dtype {info['dtype']} != U8")
blob, _ = rf.read_tensor_bytes(path, header, data_start, name)
- except rf.RansRefusal as exc:
- refuse(failures, path, name, exc.code, exc.detail)
- continue
- try:
packed = rf.decode_record(blob, table)
- except rf.RansRefusal as exc:
- refuse(failures, path, name, exc.code, exc.detail)
- continue
- # deterministic re-encode cross-check
- try:
+ # deterministic re-encode cross-check
re_rec = rf.build_record(rf.unpack_nibbles(packed), table["freq"],
table["start"], table["slot_to_symbol"],
table["scale_bits"])
+ if re_rec != blob:
+ raise rf.RansRefusal("E_REENCODE_MISMATCH",
+ "re-encoding the decoded stream does not "
+ "reproduce the record β€” content corruption")
except rf.RansRefusal as exc:
refuse(failures, path, name, exc.code, exc.detail)
continue
- if re_rec != blob:
- refuse(failures, path, name, "E_REENCODE_MISMATCH",
- "re-encoding the decoded stream does not reproduce the "
- "record β€” content corruption")
+ except Exception as exc: # noqa: BLE001 β€” the point
+ # anything unexpected becomes a named refusal: "never crashes"
+ # is enforced here, not promised
+ refuse(failures, path, name, "E_INTERNAL", repr(exc))
continue
trusted += 1
print(f"[trust] {os.path.basename(path)}: {trusted}/{checked} verified "
@@ -162,7 +169,10 @@ def main():
if not os.path.exists(p):
refuse(failures, p, None, "E_SHARD_MALFORMED", "no such file")
continue
- verify_shard(p, failures, a.max_tensors)
+ try:
+ verify_shard(p, failures, a.max_tensors)
+ except Exception as exc: # noqa: BLE001 β€” the point
+ refuse(failures, p, None, "E_INTERNAL", repr(exc))
if failures:
print(f"RESULT: REFUSE ({len(failures)} refusal(s))")
sys.exit(1)
diff --git a/c/tools/repack_rans.py b/c/tools/repack_rans.py
index 92eb304c..df87dd7d 100644
--- a/c/tools/repack_rans.py
+++ b/c/tools/repack_rans.py
@@ -59,10 +59,20 @@ import rans_format as rf # noqa: E402
EXPERT_RE = re.compile(
r"^model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate|up|down)_proj\.weight$")
-# Per-row geometry guard (see module docstring). MIN_ROW_RATIO = I/2 for the
-# smallest hidden dim this tool accepts; the load-bearing exclusion is 32,
-# the exact ratio EVERY per-group-64 container measures regardless of shape.
+# Per-row geometry guard (see module docstring). When the weight tensor
+# carries a 2-D [O, rb] shape, per-row geometry is verified POSITIVELY
+# (qs elements == O). Flattened 1-D containers (what convert_fp8_to_int4.py
+# emits) can only be judged by the weight-bytes/scale-rows ratio:
+# per-row -> ratio == I/2 (3072 / 1024 for real GLM projections)
+# per-group-64 -> ratio == exactly 32, ALWAYS, regardless of shape
+# per-group-128 (upstream fmt=4's actual gs) -> ratio == exactly 64
+# so 32 refuses as g64, 64 refuses as AMBIGUOUS (indistinguishable from a
+# per-row I=128 tensor by bytes alone), and anything below MIN_ROW_RATIO
+# refuses as not-per-row. Collateral: 1-D per-row tensors with I <= 128 are
+# refused β€” loud, by name, and vacuous for every real container this repo
+# targets.
G64_RATIO = 32
+G128_RATIO = 64
MIN_ROW_RATIO = 64
@@ -110,6 +120,18 @@ def select_targets(index, layers):
raise rf.RansRefusal("E_GEOMETRY_NOT_PER_ROW",
f"{name}: {qs} is {sb} bytes (not F32-shaped)")
rows = sb // 4
+ wshape = info["shape"]
+ if len(wshape) == 2:
+ # positive per-row verification: [O, rb] weight demands exactly
+ # one scale per row β€” no ratio heuristics involved
+ if rows != wshape[0]:
+ raise rf.RansRefusal(
+ "E_GEOMETRY_NOT_PER_ROW",
+ f"{name}: weight shape {wshape} has {wshape[0]} rows but "
+ f"{qs} carries {rows} scales β€” not per-row geometry")
+ selected.append(name)
+ continue
+ # flattened 1-D container: byte-ratio judgement (docstring above)
if wb % rows:
raise rf.RansRefusal(
"E_GEOMETRY_NOT_PER_ROW",
@@ -121,6 +143,14 @@ def select_targets(index, layers):
f"{name}: weight/scale ratio {ratio} is the per-group-64 "
f"signature β€” g64 containers are out of this format's v1 scope "
f"(materially weaker entropy payoff; needs its own round)")
+ if ratio == G128_RATIO:
+ raise rf.RansRefusal(
+ "E_GEOMETRY_AMBIGUOUS",
+ f"{name}: weight/scale ratio {ratio} is exactly the "
+ f"per-group-128 signature (upstream grouped-int4's gs) and "
+ f"indistinguishable by bytes from a per-row I=128 tensor β€” "
+ f"refusing rather than stamping per-row geometry it cannot "
+ f"prove")
if ratio < MIN_ROW_RATIO:
raise rf.RansRefusal(
"E_GEOMETRY_NOT_PER_ROW",
@@ -141,6 +171,10 @@ def main():
help="comma-separated layer subset (default: every layer)")
ap.add_argument("--dry-run", action="store_true",
help="inventory only: scan headers, print selection, write nothing")
+ ap.add_argument("--force", action="store_true",
+ help="overwrite an outdir that already contains repack output "
+ "(default: refuse β€” a pre-existing partial set from an "
+ "interrupted run must never be silently papered over)")
a = ap.parse_args()
layers = None
@@ -195,9 +229,29 @@ def main():
print(f"[table] pooled histogram {hist.tolist()}")
print(f"[table] freq {freq.tolist()} (M={rf.M}, scale_bits={rf.SCALE_BITS})")
- # pass 2: encode + verify + write, one output shard per input shard
+ # pass 2: encode + verify + write, one output shard per input shard.
+ # CRASH EVIDENCE: refuse a non-empty output directory (a crashed earlier
+ # run leaves individually-valid shards with no marker that the SET is
+ # incomplete β€” rerunning over them silently would hide that), and write
+ # repack-manifest.json only after every shard landed, so "manifest
+ # present and matching" == "the set is complete".
os.makedirs(a.outdir, exist_ok=True)
+ manifest_path = os.path.join(a.outdir, "repack-manifest.json")
+ pre_existing = sorted(glob.glob(os.path.join(a.outdir, "out-rans-*.safetensors")))
+ if (pre_existing or os.path.exists(manifest_path)) and not a.force:
+ print(f"ERROR: {a.outdir} already contains repack output "
+ f"({len(pre_existing)} shard(s)"
+ f"{', manifest present' if os.path.exists(manifest_path) else ', NO manifest β€” likely an interrupted run'}) "
+ f"β€” refusing to overwrite; pass --force to redo from scratch",
+ file=sys.stderr)
+ sys.exit(1)
+ if a.force:
+ for p in pre_existing:
+ os.remove(p)
+ if os.path.exists(manifest_path):
+ os.remove(manifest_path)
out_n = 0
+ shard_manifest = []
total_record = 0
for p in sorted(by_shard):
tensors = []
@@ -226,16 +280,37 @@ def main():
}
out_path = os.path.join(a.outdir, f"out-rans-{out_n:05d}.safetensors")
rf.write_shard(out_path, tensors, metadata)
+ shard_manifest.append({
+ "file": os.path.basename(out_path),
+ "source": os.path.basename(p),
+ "bytes": os.path.getsize(out_path),
+ "weight_tensors": len(fmt_map),
+ })
print(f"[write] {out_path}: {len(tensors)} tensor(s) "
f"({os.path.getsize(out_path) / 1e9:.4f} GB) "
f"from {os.path.basename(p)}")
out_n += 1
+ # completion marker, written LAST and deterministically (no timestamps):
+ # its absence over a shard set means the producing run did not finish
+ manifest = {
+ "format": rf.FORMAT_NAME,
+ "table_crc32": json.loads(table_json)["table_crc32"],
+ "n_shards": out_n,
+ "n_weight_tensors": len(selected),
+ "shards": shard_manifest,
+ }
+ tmp = manifest_path + ".tmp"
+ with open(tmp, "w") as fh:
+ json.dump(manifest, fh, indent=1, sort_keys=True)
+ os.replace(tmp, manifest_path)
+
ratio = total_record / total_w
print(f"[ratio] record/original: {ratio:.4f} ({(1 - ratio) * 100:.2f}% "
f"reduction incl. framing), {total_w / 1e6:.1f} MB -> "
f"{total_record / 1e6:.1f} MB")
- print(f"[done] {out_n} shard(s), every record round-trip-verified byte-exact")
+ print(f"[done] {out_n} shard(s) + repack-manifest.json, every record "
+ f"round-trip-verified byte-exact")
if __name__ == "__main__":
diff --git a/docs/int4-rans256-g0.md b/docs/int4-rans256-g0.md
index 328d0684..96df21c9 100644
--- a/docs/int4-rans256-g0.md
+++ b/docs/int4-rans256-g0.md
@@ -38,7 +38,12 @@ second codec path in v1).
### Chunk record (one record = one weight tensor's bytes)
-All integers little-endian. `N` = 256 streams.
+All integers little-endian β€” explicitly including the `n_symbols` and
+`packed_bytes` u64 fields and every `stream_offsets` u32 (the same
+byte order the base64 `slot_to_symbol` field spells out). `N` = 256
+streams. The reference C implementation reads/writes these with
+native-order accesses and therefore refuses to compile on a big-endian
+host rather than emit byte-swapped records.
```
offset 0: n_symbols u64 -- nibble count for this tensor
@@ -60,7 +65,24 @@ derived value. Validators refuse nonzero padding bytes. Alignment is
**record-relative**, not file-absolute: safetensors does not 16-align
tensor offsets, so a consumer that wants aligned u32/u64 access must read or
map the tensor's bytes into an aligned allocation (any `malloc`/GPU-buffer
-allocation qualifies).
+allocation qualifies; the reference parser refuses a buffer that is not at
+least 4-byte aligned rather than perform unaligned accesses).
+
+Validity rules every conformant record satisfies (and validators enforce):
+
+- `packed_bytes == ceil(n_symbols / 2)`, computed in a non-wrapping form
+ (`n/2 + (n & 1)`) so `n_symbols = 2^64 - 1` cannot forge a match;
+- `stream_offsets[0] == 0`, offsets non-decreasing, and **every stream is at
+ least 4 bytes** β€” even a stream that encodes zero symbols carries its
+ 4 flushed state bytes;
+- the record's total length equals the derived framing exactly (no trailing
+ bytes), and every derived padding byte is zero;
+- **amplification bound**: no admissible table can encode more than
+ `payload_len * 8 * M_max` symbols into a payload (`M_max = 2^15`, the
+ format's largest table size β€” a symbol's cost is bounded below by
+ `log2(M / (M-1))` bits). A header claiming more `n_symbols` than that is
+ a decompression bomb and must be refused *before* anything sizes buffers
+ from it.
### Interleaving
@@ -80,7 +102,18 @@ design): 32-bit state, byte renormalization, `L = 2^23`,
`scale_bits = 14` (`M = 16384`).
- encode walks the input backwards, writes bytes backwards, flushes the
- final 4-byte state; decode reads forward:
+ final 4-byte state. Per symbol `s` (this is the complete recipe a
+ bit-identical third-party **writer** needs): with the state
+ `x` (initialized to `L`), first renormalize by emitting low bytes β€”
+ `x_max = ((L >> scale_bits) << 8) * freq[s]`;
+ `while (x >= x_max) { emit byte x & 0xFF; x >>= 8; }` β€” then update
+ `x = ((x / freq[s]) << scale_bits) + (x % freq[s]) + start[s]`
+ (integer division). After the last (i.e. first-in-stream-order) symbol,
+ emit the remaining 32-bit state as 4 bytes, low byte last. The emitted
+ byte sequence, reversed into forward order, is the stream. Worst case a
+ `freq = 1` symbol emits `scale_bits` bits, so a stream of `n` symbols
+ never exceeds `ceil(n * scale_bits / 8) + 4` bytes;
+- decode reads forward:
`x = 4 bytes big-endian`, then per symbol:
`slot = x & (M-1)`; `s = slot_to_symbol[slot]`;
`x = freq[s]*(x >> scale_bits) + slot - start[s]`;