Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(model): retain exact safetensors shape dimensions
morluto0d44afed7e3f
diff --git a/docs/model-artifacts.md b/docs/model-artifacts.mdindex 5c5269b0f..63adc66c0 100644--- a/docs/model-artifacts.md+++ b/docs/model-artifacts.md@@ -65,7 +65,13 @@ contradictions are rejected with a typed `file_format_rejected` constraint (publA missing, non-regular or otherwise unusable selected path is also `target_unavailable`;host I/O or resource failures on a valid selection, such as `EMFILE` or `EIO`, are`execution_failure` with the path and reason in `details.diagnostics`. Empty-tensors and scalar shapes are supported; packed sub-byte types require whole+tensors and scalar shapes are supported. The reader retains shape dimensions as+exact unsigned 64-bit values, including values above JavaScript's safe-integer+range when a zero extent makes storage empty. Shape products are checked in+declaration order, as in the+[64-bit upstream reader](https://github.com/huggingface/safetensors/blob/e246a2560645b7525f5775669ed816eb57c5bcc8/safetensors/src/tensor.rs#L604-L609);+a later zero does not erase an earlier product overflow. Actual file and data+offsets must still fit exact host offset precision. Packed sub-byte types require wholebyte storage. Unknown dtypes retain declared ranges and an explicit limitationthat shape-to-storage validation is unavailable. Each tensor records`storage_validation: "consistent" | "unknown"`; `coverage.storage_sizes` remainsdiff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.tsindex 9fc24b211..04adfedd1 100644--- a/src/model/SafetensorsHeader.ts+++ b/src/model/SafetensorsHeader.ts@@ -17,7 +17,7 @@ import { safetensorsBits } from "./ModelTensorTypes.js";type HeaderValue =| string- | number+ | bigint| HeaderValue[]| { [key: string]: HeaderValue };@@ -100,20 +100,20 @@ class HeaderJson {}}const start = this.#position;- while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;+ while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) {+ this.#position++;+ if (this.#position - start > 20)+ rejectModel("Safetensors unsigned integer exceeds uint64");+ }const raw = this.text.slice(start, this.#position);if (!/^(?:0|[1-9][0-9]*)$/u.test(raw))return rejectModel("Safetensors header requires unsigned integer declarations",);- const number = Number(raw);- if (!Number.isSafeInteger(number))- throw new ModelFormatFailure(- "capacity",- "Safetensors integer exceeds exact host offset precision",- "exact-offset",- );- return number;+ const integer = BigInt(raw);+ if (integer > 0xffffffffffffffffn)+ rejectModel("Safetensors unsigned integer exceeds uint64");+ return integer;}*read(): Generator<void, HeaderValue> {if (this.text[0] !== "{")@@ -133,14 +133,23 @@ const objectOf = (return rejectModel("Expected safetensors header object");return value;};-const integersOf = (value: HeaderValue | undefined): number[] => {+const integersOf = (value: HeaderValue | undefined): bigint[] => {if (!Array.isArray(value) ||- !value.every((item): item is number => typeof item === "number")+ !value.every((item): item is bigint => typeof item === "bigint"))return rejectModel("Expected safetensors integer array");return value;};+const exactOffset = (value: bigint): number => {+ if (value > BigInt(Number.MAX_SAFE_INTEGER))+ throw new ModelFormatFailure(+ "capacity",+ "Safetensors data offset exceeds exact host offset precision",+ "exact-offset",+ );+ return Number(value);+};const pointer = (value: string): string =>value.replaceAll("~", "~0").replaceAll("/", "~1");@@ -243,21 +252,20 @@ export const readSafetensorsHeader = async (const offsets = integersOf(entry.data_offsets);if (offsets.length !== 2)rejectModel(`Safetensors tensor ${name} requires two data offsets`);- const start = offsets[0] ?? 0;- const end = offsets[1] ?? 0;+ const start = exactOffset(offsets[0] ?? 0n);+ const end = exactOffset(offsets[1] ?? 0n);if (end < start || end > reader.size - tensor_data_offset)rejectModel(`Safetensors tensor ${name} range [${String(start)},${String(end)}) is outside the data buffer`,);- let parameters = shape.includes(0) ? 0n : 1n;- if (parameters !== 0n)- for (const dimension of shape) {- parameters *= BigInt(dimension);- if (parameters > 0xffffffffffffffffn)- rejectModel(- `Safetensors tensor ${name} parameter count overflows uint64`,- );- }+ let parameters = 1n;+ for (const dimension of shape) {+ parameters *= dimension;+ if (parameters > 0xffffffffffffffffn)+ rejectModel(+ `Safetensors tensor ${name} parameter count overflows uint64`,+ );+ }const bits = safetensorsBits(type);if (bits !== undefined) {const expected = parameters * BigInt(bits);diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.tsindex 0b807b73f..f5b5d9d66 100644--- a/tests/acceptance/modelArtifactCli.test.ts+++ b/tests/acceptance/modelArtifactCli.test.ts@@ -19,6 +19,92 @@ import { ggufFixture, safetensorsFixture } from "../fixtures/model/model.js";import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";import { cliTest } from "../support/cli/cliFixture.js";+cliTest(+ "preserves exact large safetensors dimensions while bounding actual offsets",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-model-shape-precision-");+ const path = join(root, "empty.safetensors");+ const fixture = safetensorsFixture(+ '{"a":{"dtype":"F32","shape":[0,9007199254740992],"data_offsets":[0,0]},"b":{"dtype":"F32","shape":[18446744073709551615,0],"data_offsets":[0,0]}}',+ Buffer.alloc(0),+ );+ await writeFile(path, fixture.bytes);+ const result = await cli.run({+ arguments: [+ "inspect-model-artifact",+ path,+ "--detail",+ "complete",+ "--hash",+ "full",+ "--json",+ ],+ });+ expect(result.exitCode, result.stderr).toBe(0);+ const report = modelArtifactSchema.parse(+ toolContract("inspect_model_artifact").outputSchema.parse(result.json)+ .normalized_result,+ );+ expect(+ report.tensors.map(+ ({ shape, parameter_count, byte_length, storage_validation }) => ({+ shape,+ parameter_count,+ byte_length,+ storage_validation,+ }),+ ),+ ).toEqual([+ {+ shape: ["0", "9007199254740992"],+ parameter_count: "0",+ byte_length: 0,+ storage_validation: "consistent",+ },+ {+ shape: ["18446744073709551615", "0"],+ parameter_count: "0",+ byte_length: 0,+ storage_validation: "consistent",+ },+ ]);+ expect(report.totals).toMatchObject({+ parameter_count: "0",+ tensor_bytes: 0,+ });+ expect(report.artifact.sha256).toBe(+ createHash("sha256").update(fixture.bytes).digest("hex"),+ );+ const { call } = await connectLocalToolsMcp();+ const response = await call("inspect_model_artifact", {+ path,+ detail: "complete",+ hash: "full",+ });+ expect(response.isError, JSON.stringify(response)).not.toBe(true);+ expect(parseEvidence(response.structuredContent).normalized_result).toEqual(+ report,+ );++ await writeFile(+ path,+ safetensorsFixture(+ '{"x":{"dtype":"F32","shape":[0],"data_offsets":[9007199254740992,9007199254740992]}}',+ Buffer.alloc(0),+ ).bytes,+ );+ const offsetFailure = parseMcpToolError(+ await call("inspect_model_artifact", { path }),+ );+ expect(offsetFailure.error).toMatchObject({+ code: "resource_constraint",+ details: {+ reported_limits: { maximum_exact_offset: Number.MAX_SAFE_INTEGER },+ },+ });+ },+);+it.each(["named", "default", "inventory-only", "absent"] as const)("reports actual chat-template presence in summaries and page context (%s)",async (kind) => {diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.tsindex 9fc4ae157..35cb78959 100644--- a/tests/boundary/model/modelArtifact.test.ts+++ b/tests/boundary/model/modelArtifact.test.ts@@ -692,6 +692,30 @@ const malformed = [Buffer.alloc(4),).bytes,],+ [+ "safetensors dimension exceeding uint64",+ () =>+ safetensorsFixture(+ '{"x":{"dtype":"F32","shape":[0,18446744073709551616],"data_offsets":[0,0]}}',+ Buffer.alloc(0),+ ).bytes,+ ],+ [+ "overlong safetensors integer",+ () =>+ safetensorsFixture(+ '{"x":{"dtype":"F32","shape":[0,100000000000000000000],"data_offsets":[0,0]}}',+ Buffer.alloc(0),+ ).bytes,+ ],+ [+ "safetensors shape product overflowing before a zero extent",+ () =>+ safetensorsFixture(+ '{"x":{"dtype":"F32","shape":[18446744073709551615,18446744073709551615,0],"data_offsets":[0,0]}}',+ Buffer.alloc(0),+ ).bytes,+ ],["safetensors shape mismatch",() =>