Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(model): preserve optional metadata and tensor alignment

morluto5c7755636ddb
diff --git a/docs/model-artifacts.md b/docs/model-artifacts.md
index 63adc66c0..fc0323851 100644
--- a/docs/model-artifacts.md
+++ b/docs/model-artifacts.md
@@ -30,6 +30,9 @@ result records the exact selected path, file size, header length, header SHA-256
bytes read and read scope. GGUF alignment padding is outside this prefix.
`artifact.sha256` and the Evidence subject are null: a header digest is not a
commitment to the weights. Different files can have identical headers and sizes.
+The reported GGUF tensor-data offset is aligned after the tensor directory,
+including when the directory contains no tensors; `header_length` still ends at
+the directory, before that padding.
`hash: "full"` additionally streams the remaining file bytes through SHA-256.
It returns a full-file digest and Evidence subject, while retaining the same
@@ -59,7 +62,10 @@ declared float type; negative zero is `"-0"`. Unknown tensor types retain their
lengths; affected totals and storage validation remain unknown.
Safetensors reports the header extent, exact string `__metadata__` values, dtype,
-shape and declared data ranges. JSON duplicate keys, invalid UTF-8, out-of-file
+shape and declared data ranges. A null `__metadata__` is treated as omitted;
+unknown fields on tensor entries are ignored within the reader's 32-level JSON
+nesting limit, while the required dtype, shape and data offsets are still
+validated. JSON duplicate keys, invalid UTF-8, out-of-file
ranges, overlaps, gaps, trailing unindexed data and known dtype/shape byte-length
contradictions are rejected with a typed `file_format_rejected` constraint (public code `target_unavailable`, category `unsupported_target`).
A missing, non-regular or otherwise unusable selected path is also `target_unavailable`;
diff --git a/src/model/GgufHeader.ts b/src/model/GgufHeader.ts
index d6c05b81d..1c5fa8c61 100644
--- a/src/model/GgufHeader.ts
+++ b/src/model/GgufHeader.ts
@@ -384,9 +384,9 @@ export class GgufHeader {
progress?.onTensorDeclaration(tensor);
}
const tensor_data_offset =
- tensorCount === 0
- ? this.reader.position
- : Math.ceil(this.reader.position / alignment) * alignment;
+ Math.ceil(this.reader.position / alignment) * alignment;
+ if (tensor_data_offset > this.reader.size)
+ rejectModel("GGUF tensor data section begins outside the file");
for (const tensor of tensors) {
const offset = BigInt(tensor_data_offset) + BigInt(tensor.data_offset);
if (
diff --git a/src/model/ModelFormat.ts b/src/model/ModelFormat.ts
index 560415fb1..f6fc0dee6 100644
--- a/src/model/ModelFormat.ts
+++ b/src/model/ModelFormat.ts
@@ -5,6 +5,7 @@ export const MODEL_CAPACITY_LIMITS = [
"representation-bytes",
"exact-offset",
"metadata-nesting",
+ "json-nesting",
] as const;
/** Which fixed model allowance refused the request. */
@@ -49,6 +50,7 @@ export const MODEL_HEADER_BYTES = 100_000_000;
export const MODEL_REPORT_BYTES = 32 * 1024 * 1024;
export const MODEL_ARRAY_PREVIEW = 8;
export const MODEL_METADATA_NESTING = 32;
+export const MODEL_JSON_NESTING = 32;
const fixedModelLimit = (allowance: string): string =>
`${allowance}. This release has no runtime override. Additional host memory does not raise it. Repeating the unchanged request fails the same way.`;
@@ -74,6 +76,10 @@ export const modelCapacityRemediation = (limit: ModelCapacityLimit): string => {
return fixedModelLimit(
`GGUF metadata nesting is limited to ${String(MODEL_METADATA_NESTING)} array frames`,
);
+ case "json-nesting":
+ return fixedModelLimit(
+ `Safetensors JSON nesting is limited to ${String(MODEL_JSON_NESTING)} levels`,
+ );
}
};
@@ -92,6 +98,8 @@ export const modelCapacityReportedLimits = (
return { maximum_exact_offset: Number.MAX_SAFE_INTEGER };
case "metadata-nesting":
return { maximum_array_frames: MODEL_METADATA_NESTING };
+ case "json-nesting":
+ return { maximum_json_depth: MODEL_JSON_NESTING };
}
};
diff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.ts
index 04adfedd1..d25fc2544 100644
--- a/src/model/SafetensorsHeader.ts
+++ b/src/model/SafetensorsHeader.ts
@@ -8,6 +8,7 @@ import type {
import { ModelHeaderReader } from "./ModelHeaderReader.js";
import {
MODEL_HEADER_BYTES,
+ MODEL_JSON_NESTING,
ModelFormatFailure,
ModelReportBudget,
rejectModel,
@@ -15,9 +16,15 @@ import {
} from "./ModelFormat.js";
import { safetensorsBits } from "./ModelTensorTypes.js";
+class JsonNumber {
+ constructor(readonly raw: string) {}
+}
type HeaderValue =
| string
| bigint
+ | boolean
+ | null
+ | JsonNumber
| HeaderValue[]
| { [key: string]: HeaderValue };
@@ -64,10 +71,26 @@ class HeaderJson {
if (++this.#nodes % 4096 === 0) yield;
this.#space();
this.budget.charge(64);
- if (depth > 4)
- rejectModel("Safetensors header has an invalid nested structure");
+ if (depth > MODEL_JSON_NESTING)
+ throw new ModelFormatFailure(
+ "capacity",
+ `Safetensors JSON nesting exceeds ${String(MODEL_JSON_NESTING)} levels`,
+ "json-nesting",
+ );
const c = this.text[this.#position];
if (c === '"') return this.#string();
+ if (this.text.startsWith("null", this.#position)) {
+ this.#position += 4;
+ return null;
+ }
+ if (this.text.startsWith("true", this.#position)) {
+ this.#position += 4;
+ return true;
+ }
+ if (this.text.startsWith("false", this.#position)) {
+ this.#position += 5;
+ return false;
+ }
if (c === "{" || c === "[") {
this.#position++;
const object: { [key: string]: HeaderValue } = Object.create(null) as {
@@ -100,20 +123,37 @@ class HeaderJson {
}
}
const start = this.#position;
- while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) {
+ if (this.text[this.#position] === "-") this.#position++;
+ const integerStart = this.#position;
+ if (this.text[this.#position] === "0") this.#position++;
+ else if (/^[1-9]$/u.test(this.text[this.#position] ?? "")) {
this.#position++;
- if (this.#position - start > 20)
- rejectModel("Safetensors unsigned integer exceeds uint64");
+ while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;
}
- const raw = this.text.slice(start, this.#position);
- if (!/^(?:0|[1-9][0-9]*)$/u.test(raw))
- return rejectModel(
- "Safetensors header requires unsigned integer declarations",
- );
- const integer = BigInt(raw);
- if (integer > 0xffffffffffffffffn)
- rejectModel("Safetensors unsigned integer exceeds uint64");
- return integer;
+ if (this.#position === integerStart)
+ return rejectModel("Invalid safetensors JSON value");
+ if (this.text[this.#position] === ".") {
+ this.#position++;
+ const fractionStart = this.#position;
+ while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;
+ if (this.#position === fractionStart)
+ return rejectModel("Invalid safetensors JSON value");
+ }
+ if (/[eE]/u.test(this.text[this.#position] ?? "")) {
+ this.#position++;
+ if (/[+-]/u.test(this.text[this.#position] ?? "")) this.#position++;
+ const exponentStart = this.#position;
+ while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;
+ if (this.#position === exponentStart)
+ return rejectModel("Invalid safetensors JSON value");
+ }
+ const numeric = this.text.slice(start, this.#position);
+ if (/^(?:0|[1-9][0-9]*)$/u.test(numeric)) {
+ if (numeric.length > 20) return new JsonNumber(numeric);
+ const integer = BigInt(numeric);
+ if (integer <= 0xffffffffffffffffn) return integer;
+ }
+ return new JsonNumber(numeric);
}
*read(): Generator<void, HeaderValue> {
if (this.text[0] !== "{")
@@ -129,7 +169,13 @@ class HeaderJson {
const objectOf = (
value: HeaderValue | undefined,
): { [key: string]: HeaderValue } => {
- if (value === undefined || typeof value !== "object" || Array.isArray(value))
+ if (
+ value === undefined ||
+ value === null ||
+ typeof value !== "object" ||
+ Array.isArray(value) ||
+ value instanceof JsonNumber
+ )
return rejectModel("Expected safetensors header object");
return value;
};
@@ -211,7 +257,7 @@ export const readSafetensorsHeader = async (
const root = objectOf(step.value);
const source = { source_offset: 8, source_length: size };
const metadata: ModelMetadata[] = [];
- if (Object.hasOwn(root, "__metadata__")) {
+ if (Object.hasOwn(root, "__metadata__") && root.__metadata__ !== null) {
for (const [key, value] of Object.entries(objectOf(root.__metadata__))) {
if (typeof value !== "string")
rejectModel("Safetensors __metadata__ values must be strings");
@@ -237,7 +283,6 @@ export const readSafetensorsHeader = async (
for (const name of names) {
const entry = objectOf(root[name]);
if (
- Object.keys(entry).length !== 3 ||
!Object.hasOwn(entry, "dtype") ||
!Object.hasOwn(entry, "shape") ||
!Object.hasOwn(entry, "data_offsets")
diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts
index f5b5d9d66..feb84649b 100644
--- a/tests/acceptance/modelArtifactCli.test.ts
+++ b/tests/acceptance/modelArtifactCli.test.ts
@@ -549,6 +549,134 @@ cliTest(
},
);
+cliTest(
+ "preserves optional safetensors fields and aligns empty GGUF data in CLI and MCP",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-model-boundary-compat-");
+ const { call } = await connectLocalToolsMcp();
+ const safetensors = safetensorsFixture(
+ '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
+ Buffer.alloc(4),
+ );
+ const emptyGgufHeader = ggufFixture({ tensors: [], metadata: [] });
+ const alignedOffset = Math.ceil(emptyGgufHeader.header.length / 32) * 32;
+ const emptyGguf = Buffer.concat([
+ emptyGgufHeader.header,
+ Buffer.alloc(alignedOffset - emptyGgufHeader.header.length),
+ ]);
+
+ for (const [name, bytes] of [
+ ["optional.safetensors", safetensors.bytes],
+ ["empty.gguf", emptyGguf],
+ ] as const) {
+ const path = join(root, name);
+ await writeFile(path, bytes);
+ const result = await cli.run({
+ arguments: [
+ "inspect-model-artifact",
+ path,
+ "--detail",
+ "complete",
+ "--json",
+ ],
+ });
+ expect(result.exitCode, result.stderr).toBe(0);
+ const report = modelArtifactSchema.parse(
+ toolContract("inspect_model_artifact").outputSchema.parse(result.json)
+ .normalized_result,
+ );
+ if (name.endsWith("safetensors")) {
+ expect(report.metadata).toEqual([]);
+ expect(report.tensors[0]?.name).toBe("weight");
+ expect(report.artifact.header_sha256).toBe(
+ createHash("sha256").update(safetensors.header).digest("hex"),
+ );
+ } else {
+ expect(report.tensors).toEqual([]);
+ expect(report.tensor_data_offset).toBe(alignedOffset);
+ expect(report.artifact.header_length).toBe(
+ emptyGgufHeader.header.length,
+ );
+ expect(report.artifact.header_sha256).toBe(
+ createHash("sha256").update(emptyGgufHeader.header).digest("hex"),
+ );
+ }
+ const response = await call("inspect_model_artifact", {
+ path,
+ detail: "complete",
+ });
+ expect(response.isError, JSON.stringify(response)).not.toBe(true);
+ expect(
+ parseEvidence(response.structuredContent).normalized_result,
+ ).toEqual(report);
+ }
+
+ const mapPath = join(root, "same-key-map.safetensors");
+ await writeFile(
+ mapPath,
+ safetensorsFixture(
+ '{"__metadata__":{"json_number":"1.25"}}',
+ Buffer.alloc(0),
+ ).bytes,
+ );
+ const mapCli = await cli.run({
+ arguments: [
+ "inspect-model-artifact",
+ mapPath,
+ "--detail",
+ "complete",
+ "--json",
+ ],
+ });
+ expect(mapCli.exitCode, mapCli.stderr).toBe(0);
+ const mapReport = modelArtifactSchema.parse(
+ toolContract("inspect_model_artifact").outputSchema.parse(mapCli.json)
+ .normalized_result,
+ );
+ expect(
+ mapReport.metadata.map(({ key, value }) => ({ key, value })),
+ ).toEqual([{ key: "json_number", value: "1.25" }]);
+ const mapMcp = await call("inspect_model_artifact", {
+ path: mapPath,
+ detail: "complete",
+ });
+ expect(mapMcp.isError, JSON.stringify(mapMcp)).not.toBe(true);
+ expect(parseEvidence(mapMcp.structuredContent).normalized_result).toEqual(
+ mapReport,
+ );
+
+ for (const [name, numeric] of [
+ ["decimal", "1.25"],
+ ["negative", "-1"],
+ ["overflow", "18446744073709551616"],
+ ] as const) {
+ const malformedPath = join(root, `${name}-metadata.safetensors`);
+ await writeFile(
+ malformedPath,
+ safetensorsFixture(`{"__metadata__":${numeric}}`, Buffer.alloc(0))
+ .bytes,
+ );
+ const malformedCli = await cli.run({
+ arguments: ["inspect-model-artifact", malformedPath, "--json"],
+ });
+ expect(malformedCli.exitCode).toBe(1);
+ expect(malformedCli.json).toMatchObject({
+ code: "target_unavailable",
+ category: "unsupported_target",
+ details: { constraint: "file_format_rejected" },
+ });
+ const malformedMcp = parseMcpToolError(
+ await call("inspect_model_artifact", { path: malformedPath }),
+ );
+ expect(malformedMcp.error).toMatchObject({
+ code: "target_unavailable",
+ category: "unsupported_target",
+ details: { constraint: "file_format_rejected" },
+ });
+ }
+ },
+);
+
cliTest(
"exports complete model Evidence for portable paging and returns typed malformed errors",
async ({ cli }) => {
diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.ts
index 35cb78959..7c9a29ec9 100644
--- a/tests/boundary/model/modelArtifact.test.ts
+++ b/tests/boundary/model/modelArtifact.test.ts
@@ -5,6 +5,7 @@ import { expect, it, onTestFinished } from "vitest";
import { ModelArtifactProvider } from "../../../src/model/ModelArtifactProvider.js";
import { ModelHeaderReader } from "../../../src/model/ModelHeaderReader.js";
import {
+ MODEL_JSON_NESTING,
MODEL_METADATA_NESTING,
MODEL_REPORT_BYTES,
ModelFormatFailure,
@@ -228,6 +229,28 @@ it.each([8, 24, 40])(
},
);
+it("aligns the GGUF data section after an empty tensor directory", async () => {
+ const base = ggufFixture({ tensors: [], metadata: [] });
+ const dataOffset = Math.ceil(base.header.length / 32) * 32;
+ const bytes = Buffer.concat([
+ base.header,
+ Buffer.alloc(dataOffset - base.header.length),
+ ]);
+ const { path, provider } = await providerAt(bytes);
+ const result = await provider.inspect({ path, hash: "header" });
+ if (!result.ok) throw result.error;
+ expect(result.value.tensors).toEqual([]);
+ expect(result.value.tensor_data_offset).toBe(dataOffset);
+ expect(result.value.artifact.header_length).toBe(base.header.length);
+});
+
+it("rejects an empty GGUF whose required directory padding is truncated", async () => {
+ const fixture = ggufFixture({ tensors: [], metadata: [] });
+ const { path, provider } = await providerAt(fixture.header);
+ const result = await provider.inspect({ path, hash: "header" });
+ expect(result.ok).toBe(false);
+});
+
it.each([0, 1, 2, 4, 12])(
"rejects GGUF alignment %s outside positive multiples of eight",
async (alignment) => {
@@ -585,12 +608,18 @@ it("chooses model recovery from the producing limit", () => {
const nesting = modelCapacityRemediation("metadata-nesting");
expect(nesting).toContain(`${String(MODEL_METADATA_NESTING)} array frames`);
expect(nesting).not.toContain(String(MODEL_REPORT_BYTES));
+ const jsonNesting = modelCapacityRemediation("json-nesting");
+ expect(jsonNesting).toContain(`${String(MODEL_JSON_NESTING)} levels`);
+ expect(jsonNesting).not.toContain(String(MODEL_REPORT_BYTES));
expect(modelCapacityReportedLimits("report-bytes")).toEqual({
maximum_report_bytes: MODEL_REPORT_BYTES,
});
expect(modelCapacityReportedLimits("exact-offset")).toEqual({
maximum_exact_offset: Number.MAX_SAFE_INTEGER,
});
+ expect(modelCapacityReportedLimits("json-nesting")).toEqual({
+ maximum_json_depth: MODEL_JSON_NESTING,
+ });
});
it("refuses an oversized declared header before reading or expanding it", async () => {
@@ -742,6 +771,44 @@ const malformed = [
"safetensors non-string metadata",
() => safetensorsFixture('{"__metadata__":{"x":1}}', Buffer.alloc(0)).bytes,
],
+ [
+ "safetensors decimal metadata container",
+ () => safetensorsFixture('{"__metadata__":1.25}', Buffer.alloc(0)).bytes,
+ ],
+ [
+ "safetensors negative metadata container",
+ () => safetensorsFixture('{"__metadata__":-1}', Buffer.alloc(0)).bytes,
+ ],
+ [
+ "safetensors overflowing metadata container",
+ () =>
+ safetensorsFixture(
+ '{"__metadata__":18446744073709551616}',
+ Buffer.alloc(0),
+ ).bytes,
+ ],
+ [
+ "safetensors noninteger shape sentinel",
+ () =>
+ safetensorsFixture(
+ '{"x":{"dtype":"F32","shape":[1.25],"data_offsets":[0,0]}}',
+ Buffer.alloc(0),
+ ).bytes,
+ ],
+ [
+ "safetensors negative data-offset sentinel",
+ () =>
+ safetensorsFixture(
+ '{"x":{"dtype":"F32","shape":[1],"data_offsets":[-1,4]}}',
+ Buffer.alloc(0),
+ ).bytes,
+ ],
+ [
+ "safetensors tensor missing a required declaration",
+ () =>
+ safetensorsFixture('{"x":{"dtype":"F32","shape":[1]}}', Buffer.alloc(0))
+ .bytes,
+ ],
[
"safetensors unpaired surrogate",
() =>
@@ -762,6 +829,53 @@ it.each(malformed)(
},
);
+it("accepts null safetensors metadata and ignores tensor extension fields", async () => {
+ const fixture = safetensorsFixture(
+ '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
+ Buffer.alloc(4),
+ );
+ const { provider, path } = await providerAt(fixture.bytes);
+ const result = await provider.inspect({ path, hash: "header" });
+ if (!result.ok) throw result.error;
+ expect(result.value.metadata).toEqual([]);
+ expect(
+ result.value.tensors.map(({ name, type, shape }) => ({
+ name,
+ type,
+ shape,
+ })),
+ ).toEqual([{ name: "weight", type: "F32", shape: ["1"] }]);
+});
+
+it("accepts nested JSON values in ignored Safetensors extension fields", async () => {
+ const nested = `${"[".repeat(12)}0${"]".repeat(12)}`;
+ const fixture = safetensorsFixture(
+ `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,
+ Buffer.alloc(4),
+ );
+ const { provider, path } = await providerAt(fixture.bytes);
+ const result = await provider.inspect({ path, hash: "header" });
+ expect(result.ok).toBe(true);
+});
+
+it("classifies over-depth Safetensors extension JSON as a resource limit", async () => {
+ const nested = `${"[".repeat(MODEL_JSON_NESTING + 1)}0${"]".repeat(MODEL_JSON_NESTING + 1)}`;
+ const fixture = safetensorsFixture(
+ `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,
+ Buffer.alloc(4),
+ );
+ const { provider, path } = await providerAt(fixture.bytes);
+ const result = await provider.inspect({ path, hash: "header" });
+ expect(result.ok).toBe(false);
+ if (result.ok) throw new Error("Over-depth JSON was accepted");
+ expect(result.error).toMatchObject({
+ _tag: "AnalysisResourceConstraintError",
+ resource: "memory",
+ reason: expect.stringContaining("Safetensors JSON nesting"),
+ reportedLimits: { maximum_json_depth: MODEL_JSON_NESTING },
+ });
+});
+
it.each([640, 896, 1152, 1408, 66_176])(
"admits safetensors header length %s whose prefix bytes resemble a pickle protocol marker",
async (length) => {
diff --git a/tests/fixtures/model/model.ts b/tests/fixtures/model/model.ts
index c73d25c2e..012bb7865 100644
--- a/tests/fixtures/model/model.ts
+++ b/tests/fixtures/model/model.ts
@@ -77,10 +77,7 @@ export const ggufFixture = (
]),
),
]);
- const dataOffset =
- tensors.length === 0
- ? header.length
- : Math.ceil(header.length / alignment) * alignment;
+ const dataOffset = Math.ceil(header.length / alignment) * alignment;
const length = Math.max(
dataOffset,
...tensors.map((t) => dataOffset + Number(t.offset) + t.bytes),