Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(model): retain exact safetensors shape dimensions

morluto0d44afed7e3f
diff --git a/docs/model-artifacts.md b/docs/model-artifacts.md
index 5c5269b0f..63adc66c0 100644
--- a/docs/model-artifacts.md
+++ b/docs/model-artifacts.md
@@ -65,7 +65,13 @@ contradictions are rejected with a typed `file_format_rejected` constraint (publ
A missing, non-regular or otherwise unusable selected path is also `target_unavailable`;
host I/O or resource failures on a valid selection, such as `EMFILE` or `EIO`, are
`execution_failure` with the path and reason in `details.diagnostics`. Empty
-tensors and scalar shapes are supported; packed sub-byte types require whole
+tensors and scalar shapes are supported. The reader retains shape dimensions as
+exact unsigned 64-bit values, including values above JavaScript's safe-integer
+range when a zero extent makes storage empty. Shape products are checked in
+declaration order, as in the
+[64-bit upstream reader](https://github.com/huggingface/safetensors/blob/e246a2560645b7525f5775669ed816eb57c5bcc8/safetensors/src/tensor.rs#L604-L609);
+a later zero does not erase an earlier product overflow. Actual file and data
+offsets must still fit exact host offset precision. Packed sub-byte types require whole
byte storage. Unknown dtypes retain declared ranges and an explicit limitation
that shape-to-storage validation is unavailable. Each tensor records
`storage_validation: "consistent" | "unknown"`; `coverage.storage_sizes` remains
diff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.ts
index 9fc24b211..04adfedd1 100644
--- a/src/model/SafetensorsHeader.ts
+++ b/src/model/SafetensorsHeader.ts
@@ -17,7 +17,7 @@ import { safetensorsBits } from "./ModelTensorTypes.js";
type HeaderValue =
| string
- | number
+ | bigint
| HeaderValue[]
| { [key: string]: HeaderValue };
@@ -100,20 +100,20 @@ class HeaderJson {
}
}
const start = this.#position;
- while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;
+ while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) {
+ this.#position++;
+ if (this.#position - start > 20)
+ rejectModel("Safetensors unsigned integer exceeds uint64");
+ }
const raw = this.text.slice(start, this.#position);
if (!/^(?:0|[1-9][0-9]*)$/u.test(raw))
return rejectModel(
"Safetensors header requires unsigned integer declarations",
);
- const number = Number(raw);
- if (!Number.isSafeInteger(number))
- throw new ModelFormatFailure(
- "capacity",
- "Safetensors integer exceeds exact host offset precision",
- "exact-offset",
- );
- return number;
+ const integer = BigInt(raw);
+ if (integer > 0xffffffffffffffffn)
+ rejectModel("Safetensors unsigned integer exceeds uint64");
+ return integer;
}
*read(): Generator<void, HeaderValue> {
if (this.text[0] !== "{")
@@ -133,14 +133,23 @@ const objectOf = (
return rejectModel("Expected safetensors header object");
return value;
};
-const integersOf = (value: HeaderValue | undefined): number[] => {
+const integersOf = (value: HeaderValue | undefined): bigint[] => {
if (
!Array.isArray(value) ||
- !value.every((item): item is number => typeof item === "number")
+ !value.every((item): item is bigint => typeof item === "bigint")
)
return rejectModel("Expected safetensors integer array");
return value;
};
+const exactOffset = (value: bigint): number => {
+ if (value > BigInt(Number.MAX_SAFE_INTEGER))
+ throw new ModelFormatFailure(
+ "capacity",
+ "Safetensors data offset exceeds exact host offset precision",
+ "exact-offset",
+ );
+ return Number(value);
+};
const pointer = (value: string): string =>
value.replaceAll("~", "~0").replaceAll("/", "~1");
@@ -243,21 +252,20 @@ export const readSafetensorsHeader = async (
const offsets = integersOf(entry.data_offsets);
if (offsets.length !== 2)
rejectModel(`Safetensors tensor ${name} requires two data offsets`);
- const start = offsets[0] ?? 0;
- const end = offsets[1] ?? 0;
+ const start = exactOffset(offsets[0] ?? 0n);
+ const end = exactOffset(offsets[1] ?? 0n);
if (end < start || end > reader.size - tensor_data_offset)
rejectModel(
`Safetensors tensor ${name} range [${String(start)},${String(end)}) is outside the data buffer`,
);
- let parameters = shape.includes(0) ? 0n : 1n;
- if (parameters !== 0n)
- for (const dimension of shape) {
- parameters *= BigInt(dimension);
- if (parameters > 0xffffffffffffffffn)
- rejectModel(
- `Safetensors tensor ${name} parameter count overflows uint64`,
- );
- }
+ let parameters = 1n;
+ for (const dimension of shape) {
+ parameters *= dimension;
+ if (parameters > 0xffffffffffffffffn)
+ rejectModel(
+ `Safetensors tensor ${name} parameter count overflows uint64`,
+ );
+ }
const bits = safetensorsBits(type);
if (bits !== undefined) {
const expected = parameters * BigInt(bits);
diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts
index 0b807b73f..f5b5d9d66 100644
--- a/tests/acceptance/modelArtifactCli.test.ts
+++ b/tests/acceptance/modelArtifactCli.test.ts
@@ -19,6 +19,92 @@ import { ggufFixture, safetensorsFixture } from "../fixtures/model/model.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { cliTest } from "../support/cli/cliFixture.js";
+cliTest(
+ "preserves exact large safetensors dimensions while bounding actual offsets",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-model-shape-precision-");
+ const path = join(root, "empty.safetensors");
+ const fixture = safetensorsFixture(
+ '{"a":{"dtype":"F32","shape":[0,9007199254740992],"data_offsets":[0,0]},"b":{"dtype":"F32","shape":[18446744073709551615,0],"data_offsets":[0,0]}}',
+ Buffer.alloc(0),
+ );
+ await writeFile(path, fixture.bytes);
+ const result = await cli.run({
+ arguments: [
+ "inspect-model-artifact",
+ path,
+ "--detail",
+ "complete",
+ "--hash",
+ "full",
+ "--json",
+ ],
+ });
+ expect(result.exitCode, result.stderr).toBe(0);
+ const report = modelArtifactSchema.parse(
+ toolContract("inspect_model_artifact").outputSchema.parse(result.json)
+ .normalized_result,
+ );
+ expect(
+ report.tensors.map(
+ ({ shape, parameter_count, byte_length, storage_validation }) => ({
+ shape,
+ parameter_count,
+ byte_length,
+ storage_validation,
+ }),
+ ),
+ ).toEqual([
+ {
+ shape: ["0", "9007199254740992"],
+ parameter_count: "0",
+ byte_length: 0,
+ storage_validation: "consistent",
+ },
+ {
+ shape: ["18446744073709551615", "0"],
+ parameter_count: "0",
+ byte_length: 0,
+ storage_validation: "consistent",
+ },
+ ]);
+ expect(report.totals).toMatchObject({
+ parameter_count: "0",
+ tensor_bytes: 0,
+ });
+ expect(report.artifact.sha256).toBe(
+ createHash("sha256").update(fixture.bytes).digest("hex"),
+ );
+ const { call } = await connectLocalToolsMcp();
+ const response = await call("inspect_model_artifact", {
+ path,
+ detail: "complete",
+ hash: "full",
+ });
+ expect(response.isError, JSON.stringify(response)).not.toBe(true);
+ expect(parseEvidence(response.structuredContent).normalized_result).toEqual(
+ report,
+ );
+
+ await writeFile(
+ path,
+ safetensorsFixture(
+ '{"x":{"dtype":"F32","shape":[0],"data_offsets":[9007199254740992,9007199254740992]}}',
+ Buffer.alloc(0),
+ ).bytes,
+ );
+ const offsetFailure = parseMcpToolError(
+ await call("inspect_model_artifact", { path }),
+ );
+ expect(offsetFailure.error).toMatchObject({
+ code: "resource_constraint",
+ details: {
+ reported_limits: { maximum_exact_offset: Number.MAX_SAFE_INTEGER },
+ },
+ });
+ },
+);
+
it.each(["named", "default", "inventory-only", "absent"] as const)(
"reports actual chat-template presence in summaries and page context (%s)",
async (kind) => {
diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts
index 9fc4ae157..35cb78959 100644
--- a/tests/boundary/model/modelArtifact.test.ts
+++ b/tests/boundary/model/modelArtifact.test.ts
@@ -692,6 +692,30 @@ const malformed = [
Buffer.alloc(4),
).bytes,
],
+ [
+ "safetensors dimension exceeding uint64",
+ () =>
+ safetensorsFixture(
+ '{"x":{"dtype":"F32","shape":[0,18446744073709551616],"data_offsets":[0,0]}}',
+ Buffer.alloc(0),
+ ).bytes,
+ ],
+ [
+ "overlong safetensors integer",
+ () =>
+ safetensorsFixture(
+ '{"x":{"dtype":"F32","shape":[0,100000000000000000000],"data_offsets":[0,0]}}',
+ Buffer.alloc(0),
+ ).bytes,
+ ],
+ [
+ "safetensors shape product overflowing before a zero extent",
+ () =>
+ safetensorsFixture(
+ '{"x":{"dtype":"F32","shape":[18446744073709551615,18446744073709551615,0],"data_offsets":[0,0]}}',
+ Buffer.alloc(0),
+ ).bytes,
+ ],
[
"safetensors shape mismatch",
() =>