Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(model): preserve optional metadata and tensor alignment
morluto5c7755636ddb
diff --git a/docs/model-artifacts.md b/docs/model-artifacts.mdindex 63adc66c0..fc0323851 100644--- a/docs/model-artifacts.md+++ b/docs/model-artifacts.md@@ -30,6 +30,9 @@ result records the exact selected path, file size, header length, header SHA-256bytes read and read scope. GGUF alignment padding is outside this prefix.`artifact.sha256` and the Evidence subject are null: a header digest is not acommitment to the weights. Different files can have identical headers and sizes.+The reported GGUF tensor-data offset is aligned after the tensor directory,+including when the directory contains no tensors; `header_length` still ends at+the directory, before that padding.`hash: "full"` additionally streams the remaining file bytes through SHA-256.It returns a full-file digest and Evidence subject, while retaining the same@@ -59,7 +62,10 @@ declared float type; negative zero is `"-0"`. Unknown tensor types retain theirlengths; affected totals and storage validation remain unknown.Safetensors reports the header extent, exact string `__metadata__` values, dtype,-shape and declared data ranges. JSON duplicate keys, invalid UTF-8, out-of-file+shape and declared data ranges. A null `__metadata__` is treated as omitted;+unknown fields on tensor entries are ignored within the reader's 32-level JSON+nesting limit, while the required dtype, shape and data offsets are still+validated. JSON duplicate keys, invalid UTF-8, out-of-fileranges, overlaps, gaps, trailing unindexed data and known dtype/shape byte-lengthcontradictions are rejected with a typed `file_format_rejected` constraint (public code `target_unavailable`, category `unsupported_target`).A missing, non-regular or otherwise unusable selected path is also `target_unavailable`;diff --git a/src/model/GgufHeader.ts b/src/model/GgufHeader.tsindex d6c05b81d..1c5fa8c61 100644--- a/src/model/GgufHeader.ts+++ b/src/model/GgufHeader.ts@@ -384,9 +384,9 @@ export class GgufHeader {progress?.onTensorDeclaration(tensor);}const tensor_data_offset =- tensorCount === 0- ? this.reader.position- : Math.ceil(this.reader.position / alignment) * alignment;+ Math.ceil(this.reader.position / alignment) * alignment;+ if (tensor_data_offset > this.reader.size)+ rejectModel("GGUF tensor data section begins outside the file");for (const tensor of tensors) {const offset = BigInt(tensor_data_offset) + BigInt(tensor.data_offset);if (diff --git a/src/model/ModelFormat.ts b/src/model/ModelFormat.tsindex 560415fb1..f6fc0dee6 100644--- a/src/model/ModelFormat.ts+++ b/src/model/ModelFormat.ts@@ -5,6 +5,7 @@ export const MODEL_CAPACITY_LIMITS = ["representation-bytes","exact-offset","metadata-nesting",+ "json-nesting",] as const;/** Which fixed model allowance refused the request. */@@ -49,6 +50,7 @@ export const MODEL_HEADER_BYTES = 100_000_000;export const MODEL_REPORT_BYTES = 32 * 1024 * 1024;export const MODEL_ARRAY_PREVIEW = 8;export const MODEL_METADATA_NESTING = 32;+export const MODEL_JSON_NESTING = 32;const fixedModelLimit = (allowance: string): string =>`${allowance}. This release has no runtime override. Additional host memory does not raise it. Repeating the unchanged request fails the same way.`;@@ -74,6 +76,10 @@ export const modelCapacityRemediation = (limit: ModelCapacityLimit): string => {return fixedModelLimit(`GGUF metadata nesting is limited to ${String(MODEL_METADATA_NESTING)} array frames`,);+ case "json-nesting":+ return fixedModelLimit(+ `Safetensors JSON nesting is limited to ${String(MODEL_JSON_NESTING)} levels`,+ );}};@@ -92,6 +98,8 @@ export const modelCapacityReportedLimits = (return { maximum_exact_offset: Number.MAX_SAFE_INTEGER };case "metadata-nesting":return { maximum_array_frames: MODEL_METADATA_NESTING };+ case "json-nesting":+ return { maximum_json_depth: MODEL_JSON_NESTING };}};diff --git a/src/model/SafetensorsHeader.ts b/src/model/SafetensorsHeader.tsindex 04adfedd1..d25fc2544 100644--- a/src/model/SafetensorsHeader.ts+++ b/src/model/SafetensorsHeader.ts@@ -8,6 +8,7 @@ import type {import { ModelHeaderReader } from "./ModelHeaderReader.js";import {MODEL_HEADER_BYTES,+ MODEL_JSON_NESTING,ModelFormatFailure,ModelReportBudget,rejectModel,@@ -15,9 +16,15 @@ import {} from "./ModelFormat.js";import { safetensorsBits } from "./ModelTensorTypes.js";+class JsonNumber {+ constructor(readonly raw: string) {}+}type HeaderValue =| string| bigint+ | boolean+ | null+ | JsonNumber| HeaderValue[]| { [key: string]: HeaderValue };@@ -64,10 +71,26 @@ class HeaderJson {if (++this.#nodes % 4096 === 0) yield;this.#space();this.budget.charge(64);- if (depth > 4)- rejectModel("Safetensors header has an invalid nested structure");+ if (depth > MODEL_JSON_NESTING)+ throw new ModelFormatFailure(+ "capacity",+ `Safetensors JSON nesting exceeds ${String(MODEL_JSON_NESTING)} levels`,+ "json-nesting",+ );const c = this.text[this.#position];if (c === '"') return this.#string();+ if (this.text.startsWith("null", this.#position)) {+ this.#position += 4;+ return null;+ }+ if (this.text.startsWith("true", this.#position)) {+ this.#position += 4;+ return true;+ }+ if (this.text.startsWith("false", this.#position)) {+ this.#position += 5;+ return false;+ }if (c === "{" || c === "[") {this.#position++;const object: { [key: string]: HeaderValue } = Object.create(null) as {@@ -100,20 +123,37 @@ class HeaderJson {}}const start = this.#position;- while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) {+ if (this.text[this.#position] === "-") this.#position++;+ const integerStart = this.#position;+ if (this.text[this.#position] === "0") this.#position++;+ else if (/^[1-9]$/u.test(this.text[this.#position] ?? "")) {this.#position++;- if (this.#position - start > 20)- rejectModel("Safetensors unsigned integer exceeds uint64");+ while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;}- const raw = this.text.slice(start, this.#position);- if (!/^(?:0|[1-9][0-9]*)$/u.test(raw))- return rejectModel(- "Safetensors header requires unsigned integer declarations",- );- const integer = BigInt(raw);- if (integer > 0xffffffffffffffffn)- rejectModel("Safetensors unsigned integer exceeds uint64");- return integer;+ if (this.#position === integerStart)+ return rejectModel("Invalid safetensors JSON value");+ if (this.text[this.#position] === ".") {+ this.#position++;+ const fractionStart = this.#position;+ while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;+ if (this.#position === fractionStart)+ return rejectModel("Invalid safetensors JSON value");+ }+ if (/[eE]/u.test(this.text[this.#position] ?? "")) {+ this.#position++;+ if (/[+-]/u.test(this.text[this.#position] ?? "")) this.#position++;+ const exponentStart = this.#position;+ while (/^[0-9]$/u.test(this.text[this.#position] ?? "")) this.#position++;+ if (this.#position === exponentStart)+ return rejectModel("Invalid safetensors JSON value");+ }+ const numeric = this.text.slice(start, this.#position);+ if (/^(?:0|[1-9][0-9]*)$/u.test(numeric)) {+ if (numeric.length > 20) return new JsonNumber(numeric);+ const integer = BigInt(numeric);+ if (integer <= 0xffffffffffffffffn) return integer;+ }+ return new JsonNumber(numeric);}*read(): Generator<void, HeaderValue> {if (this.text[0] !== "{")@@ -129,7 +169,13 @@ class HeaderJson {const objectOf = (value: HeaderValue | undefined,): { [key: string]: HeaderValue } => {- if (value === undefined || typeof value !== "object" || Array.isArray(value))+ if (+ value === undefined ||+ value === null ||+ typeof value !== "object" ||+ Array.isArray(value) ||+ value instanceof JsonNumber+ )return rejectModel("Expected safetensors header object");return value;};@@ -211,7 +257,7 @@ export const readSafetensorsHeader = async (const root = objectOf(step.value);const source = { source_offset: 8, source_length: size };const metadata: ModelMetadata[] = [];- if (Object.hasOwn(root, "__metadata__")) {+ if (Object.hasOwn(root, "__metadata__") && root.__metadata__ !== null) {for (const [key, value] of Object.entries(objectOf(root.__metadata__))) {if (typeof value !== "string")rejectModel("Safetensors __metadata__ values must be strings");@@ -237,7 +283,6 @@ export const readSafetensorsHeader = async (for (const name of names) {const entry = objectOf(root[name]);if (- Object.keys(entry).length !== 3 ||!Object.hasOwn(entry, "dtype") ||!Object.hasOwn(entry, "shape") ||!Object.hasOwn(entry, "data_offsets")diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.tsindex f5b5d9d66..feb84649b 100644--- a/tests/acceptance/modelArtifactCli.test.ts+++ b/tests/acceptance/modelArtifactCli.test.ts@@ -549,6 +549,134 @@ cliTest(},);+cliTest(+ "preserves optional safetensors fields and aligns empty GGUF data in CLI and MCP",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-model-boundary-compat-");+ const { call } = await connectLocalToolsMcp();+ const safetensors = safetensorsFixture(+ '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',+ Buffer.alloc(4),+ );+ const emptyGgufHeader = ggufFixture({ tensors: [], metadata: [] });+ const alignedOffset = Math.ceil(emptyGgufHeader.header.length / 32) * 32;+ const emptyGguf = Buffer.concat([+ emptyGgufHeader.header,+ Buffer.alloc(alignedOffset - emptyGgufHeader.header.length),+ ]);++ for (const [name, bytes] of [+ ["optional.safetensors", safetensors.bytes],+ ["empty.gguf", emptyGguf],+ ] as const) {+ const path = join(root, name);+ await writeFile(path, bytes);+ const result = await cli.run({+ arguments: [+ "inspect-model-artifact",+ path,+ "--detail",+ "complete",+ "--json",+ ],+ });+ expect(result.exitCode, result.stderr).toBe(0);+ const report = modelArtifactSchema.parse(+ toolContract("inspect_model_artifact").outputSchema.parse(result.json)+ .normalized_result,+ );+ if (name.endsWith("safetensors")) {+ expect(report.metadata).toEqual([]);+ expect(report.tensors[0]?.name).toBe("weight");+ expect(report.artifact.header_sha256).toBe(+ createHash("sha256").update(safetensors.header).digest("hex"),+ );+ } else {+ expect(report.tensors).toEqual([]);+ expect(report.tensor_data_offset).toBe(alignedOffset);+ expect(report.artifact.header_length).toBe(+ emptyGgufHeader.header.length,+ );+ expect(report.artifact.header_sha256).toBe(+ createHash("sha256").update(emptyGgufHeader.header).digest("hex"),+ );+ }+ const response = await call("inspect_model_artifact", {+ path,+ detail: "complete",+ });+ expect(response.isError, JSON.stringify(response)).not.toBe(true);+ expect(+ parseEvidence(response.structuredContent).normalized_result,+ ).toEqual(report);+ }++ const mapPath = join(root, "same-key-map.safetensors");+ await writeFile(+ mapPath,+ safetensorsFixture(+ '{"__metadata__":{"json_number":"1.25"}}',+ Buffer.alloc(0),+ ).bytes,+ );+ const mapCli = await cli.run({+ arguments: [+ "inspect-model-artifact",+ mapPath,+ "--detail",+ "complete",+ "--json",+ ],+ });+ expect(mapCli.exitCode, mapCli.stderr).toBe(0);+ const mapReport = modelArtifactSchema.parse(+ toolContract("inspect_model_artifact").outputSchema.parse(mapCli.json)+ .normalized_result,+ );+ expect(+ mapReport.metadata.map(({ key, value }) => ({ key, value })),+ ).toEqual([{ key: "json_number", value: "1.25" }]);+ const mapMcp = await call("inspect_model_artifact", {+ path: mapPath,+ detail: "complete",+ });+ expect(mapMcp.isError, JSON.stringify(mapMcp)).not.toBe(true);+ expect(parseEvidence(mapMcp.structuredContent).normalized_result).toEqual(+ mapReport,+ );++ for (const [name, numeric] of [+ ["decimal", "1.25"],+ ["negative", "-1"],+ ["overflow", "18446744073709551616"],+ ] as const) {+ const malformedPath = join(root, `${name}-metadata.safetensors`);+ await writeFile(+ malformedPath,+ safetensorsFixture(`{"__metadata__":${numeric}}`, Buffer.alloc(0))+ .bytes,+ );+ const malformedCli = await cli.run({+ arguments: ["inspect-model-artifact", malformedPath, "--json"],+ });+ expect(malformedCli.exitCode).toBe(1);+ expect(malformedCli.json).toMatchObject({+ code: "target_unavailable",+ category: "unsupported_target",+ details: { constraint: "file_format_rejected" },+ });+ const malformedMcp = parseMcpToolError(+ await call("inspect_model_artifact", { path: malformedPath }),+ );+ expect(malformedMcp.error).toMatchObject({+ code: "target_unavailable",+ category: "unsupported_target",+ details: { constraint: "file_format_rejected" },+ });+ }+ },+);+cliTest("exports complete model Evidence for portable paging and returns typed malformed errors",async ({ cli }) => {diff --git a/tests/boundary/model/modelArtifact.test.ts b/tests/boundary/model/modelArtifact.test.tsindex 35cb78959..7c9a29ec9 100644--- a/tests/boundary/model/modelArtifact.test.ts+++ b/tests/boundary/model/modelArtifact.test.ts@@ -5,6 +5,7 @@ import { expect, it, onTestFinished } from "vitest";import { ModelArtifactProvider } from "../../../src/model/ModelArtifactProvider.js";import { ModelHeaderReader } from "../../../src/model/ModelHeaderReader.js";import {+ MODEL_JSON_NESTING,MODEL_METADATA_NESTING,MODEL_REPORT_BYTES,ModelFormatFailure,@@ -228,6 +229,28 @@ it.each([8, 24, 40])(},);+it("aligns the GGUF data section after an empty tensor directory", async () => {+ const base = ggufFixture({ tensors: [], metadata: [] });+ const dataOffset = Math.ceil(base.header.length / 32) * 32;+ const bytes = Buffer.concat([+ base.header,+ Buffer.alloc(dataOffset - base.header.length),+ ]);+ const { path, provider } = await providerAt(bytes);+ const result = await provider.inspect({ path, hash: "header" });+ if (!result.ok) throw result.error;+ expect(result.value.tensors).toEqual([]);+ expect(result.value.tensor_data_offset).toBe(dataOffset);+ expect(result.value.artifact.header_length).toBe(base.header.length);+});++it("rejects an empty GGUF whose required directory padding is truncated", async () => {+ const fixture = ggufFixture({ tensors: [], metadata: [] });+ const { path, provider } = await providerAt(fixture.header);+ const result = await provider.inspect({ path, hash: "header" });+ expect(result.ok).toBe(false);+});+it.each([0, 1, 2, 4, 12])("rejects GGUF alignment %s outside positive multiples of eight",async (alignment) => {@@ -585,12 +608,18 @@ it("chooses model recovery from the producing limit", () => {const nesting = modelCapacityRemediation("metadata-nesting");expect(nesting).toContain(`${String(MODEL_METADATA_NESTING)} array frames`);expect(nesting).not.toContain(String(MODEL_REPORT_BYTES));+ const jsonNesting = modelCapacityRemediation("json-nesting");+ expect(jsonNesting).toContain(`${String(MODEL_JSON_NESTING)} levels`);+ expect(jsonNesting).not.toContain(String(MODEL_REPORT_BYTES));expect(modelCapacityReportedLimits("report-bytes")).toEqual({maximum_report_bytes: MODEL_REPORT_BYTES,});expect(modelCapacityReportedLimits("exact-offset")).toEqual({maximum_exact_offset: Number.MAX_SAFE_INTEGER,});+ expect(modelCapacityReportedLimits("json-nesting")).toEqual({+ maximum_json_depth: MODEL_JSON_NESTING,+ });});it("refuses an oversized declared header before reading or expanding it", async () => {@@ -742,6 +771,44 @@ const malformed = ["safetensors non-string metadata",() => safetensorsFixture('{"__metadata__":{"x":1}}', Buffer.alloc(0)).bytes,],+ [+ "safetensors decimal metadata container",+ () => safetensorsFixture('{"__metadata__":1.25}', Buffer.alloc(0)).bytes,+ ],+ [+ "safetensors negative metadata container",+ () => safetensorsFixture('{"__metadata__":-1}', Buffer.alloc(0)).bytes,+ ],+ [+ "safetensors overflowing metadata container",+ () =>+ safetensorsFixture(+ '{"__metadata__":18446744073709551616}',+ Buffer.alloc(0),+ ).bytes,+ ],+ [+ "safetensors noninteger shape sentinel",+ () =>+ safetensorsFixture(+ '{"x":{"dtype":"F32","shape":[1.25],"data_offsets":[0,0]}}',+ Buffer.alloc(0),+ ).bytes,+ ],+ [+ "safetensors negative data-offset sentinel",+ () =>+ safetensorsFixture(+ '{"x":{"dtype":"F32","shape":[1],"data_offsets":[-1,4]}}',+ Buffer.alloc(0),+ ).bytes,+ ],+ [+ "safetensors tensor missing a required declaration",+ () =>+ safetensorsFixture('{"x":{"dtype":"F32","shape":[1]}}', Buffer.alloc(0))+ .bytes,+ ],["safetensors unpaired surrogate",() =>@@ -762,6 +829,53 @@ it.each(malformed)(},);+it("accepts null safetensors metadata and ignores tensor extension fields", async () => {+ const fixture = safetensorsFixture(+ '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',+ Buffer.alloc(4),+ );+ const { provider, path } = await providerAt(fixture.bytes);+ const result = await provider.inspect({ path, hash: "header" });+ if (!result.ok) throw result.error;+ expect(result.value.metadata).toEqual([]);+ expect(+ result.value.tensors.map(({ name, type, shape }) => ({+ name,+ type,+ shape,+ })),+ ).toEqual([{ name: "weight", type: "F32", shape: ["1"] }]);+});++it("accepts nested JSON values in ignored Safetensors extension fields", async () => {+ const nested = `${"[".repeat(12)}0${"]".repeat(12)}`;+ const fixture = safetensorsFixture(+ `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,+ Buffer.alloc(4),+ );+ const { provider, path } = await providerAt(fixture.bytes);+ const result = await provider.inspect({ path, hash: "header" });+ expect(result.ok).toBe(true);+});++it("classifies over-depth Safetensors extension JSON as a resource limit", async () => {+ const nested = `${"[".repeat(MODEL_JSON_NESTING + 1)}0${"]".repeat(MODEL_JSON_NESTING + 1)}`;+ const fixture = safetensorsFixture(+ `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,+ Buffer.alloc(4),+ );+ const { provider, path } = await providerAt(fixture.bytes);+ const result = await provider.inspect({ path, hash: "header" });+ expect(result.ok).toBe(false);+ if (result.ok) throw new Error("Over-depth JSON was accepted");+ expect(result.error).toMatchObject({+ _tag: "AnalysisResourceConstraintError",+ resource: "memory",+ reason: expect.stringContaining("Safetensors JSON nesting"),+ reportedLimits: { maximum_json_depth: MODEL_JSON_NESTING },+ });+});+it.each([640, 896, 1152, 1408, 66_176])("admits safetensors header length %s whose prefix bytes resemble a pickle protocol marker",async (length) => {diff --git a/tests/fixtures/model/model.ts b/tests/fixtures/model/model.tsindex c73d25c2e..012bb7865 100644--- a/tests/fixtures/model/model.ts+++ b/tests/fixtures/model/model.ts@@ -77,10 +77,7 @@ export const ggufFixture = (]),),]);- const dataOffset =- tensors.length === 0- ? header.length- : Math.ceil(header.length / alignment) * alignment;+ const dataOffset = Math.ceil(header.length / alignment) * alignment;const length = Math.max(dataOffset,...tensors.map((t) => dataOffset + Number(t.offset) + t.bytes),