Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

39.6 KB
Raw Blame
import { createHash } from "node:crypto";
import { open, writeFile, symlink, truncate } from "node:fs/promises";
import { join } from "node:path";
import { expect, it, onTestFinished } from "vitest";
import { ModelArtifactProvider } from "../../../src/model/ModelArtifactProvider.js";
import { ModelHeaderReader } from "../../../src/model/ModelHeaderReader.js";
import {
  MODEL_JSON_NESTING,
  MODEL_METADATA_NESTING,
  MODEL_REPORT_BYTES,
  ModelFormatFailure,
  modelCapacityRemediation,
  modelCapacityReportedLimits,
} from "../../../src/model/ModelFormat.js";
import { readSafetensorsHeader } from "../../../src/model/SafetensorsHeader.js";
import { GgufHeader } from "../../../src/model/GgufHeader.js";
import { createModelArtifactService } from "../../../src/composition/model.js";
import { modelArtifactSchema } from "../../../src/domain/model/modelArtifact.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { ggufFixture, safetensorsFixture } from "../../fixtures/model/model.js";

const hash = (bytes: Buffer) =>
  createHash("sha256").update(bytes).digest("hex");
const providerAt = async (bytes: Buffer) => {
  const root = await createTestTempDirectory("rea-model-reader-");
  const path = join(root, "content.without-format-extension");
  await writeFile(path, bytes);
  const provider = new ModelArtifactProvider();
  onTestFinished(() => provider.close());
  return { provider, path, root };
};

it.each([
  ["01000000", 1, 16_777_216],
  ["00000001", 16_777_216, 1],
  ["04000000", 4, 67_108_864],
  ["00000004", 67_108_864, 4],
  ["e7030000", 999, 3_875_733_504],
  ["000003e7", 3_875_733_504, 999],
  ["00000000", 0, 0],
  ["ffffffff", 4_294_967_295, 4_294_967_295],
] as const)(
  "preserves unsupported GGUF version bytes %s without assuming byte order",
  async (versionBytes, little, big) => {
    // No counts are needed to explain a version refusal. Do not reinterpret
    // an unknown version's subsequent layout or publish a header observation.
    const { provider, path } = await providerAt(
      Buffer.from(`47475546${versionBytes}`, "hex"),
    );
    const result = await provider.inspect({ path, hash: "header" });
    if (result.ok) throw new Error("Unknown version admitted");
    expect(result.error).toMatchObject({
      _tag: "AnalysisUnsupportedTargetError",
      reason: `GGUF version bytes ${versionBytes} are unsupported (little-endian ${String(little)}, big-endian ${String(big)}; byte order unknown); supported versions are 2 and 3`,
    });
  },
);

it("continues reading little-endian GGUF v2 declarations", async () => {
  const { provider, path } = await providerAt(
    ggufFixture({ version: 2 }).bytes,
  );
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value).toMatchObject({ version: 2, byte_order: "little" });
  expect(result.value.tensors.map(({ name }) => name)).toEqual([
    "weight",
    "quant",
  ]);
});

it("continues refusing the unsupported big-endian GGUF v2 layout", async () => {
  const { provider, path } = await providerAt(
    ggufFixture({ version: 2, bigEndian: true }).bytes,
  );
  const result = await provider.inspect({ path, hash: "header" });
  if (result.ok) throw new Error("Big-endian GGUF v2 admitted");
  expect(result.error).toMatchObject({
    _tag: "BinaryTargetError",
    constraint: "file_format_rejected",
  });
});

it.each([false, true])(
  "reads exact GGUF declarations, endian=%s, without hashing tensor payload",
  async (bigEndian) => {
    const fixture = ggufFixture({ bigEndian });
    const { provider, path } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (!result.ok) throw result.error;
    expect(result.value.artifact).toEqual({
      path,
      file_size: fixture.bytes.length,
      header_sha256: hash(fixture.header),
      header_length: fixture.header.length,
      sha256: null,
      bytes_read: fixture.header.length,
      read_scope: "header",
    });
    expect(result.value.byte_order).toBe(bigEndian ? "big" : "little");
    expect(
      result.value.metadata.find((row) => row.key === "custom.exact")?.value,
    ).toBe("18446744073709551615");
    expect(
      result.value.metadata.find((row) => row.key === "tokenizer.ggml.tokens"),
    ).toMatchObject({
      element_type: "STRING",
      element_count: 10,
      preview_complete: false,
      value: Array.from({ length: 8 }, (_, n) => `token-${String(n)}`),
    });
    expect(
      result.value.tensors.map(({ name, shape, byte_length, file_offset }) => ({
        name,
        shape,
        byte_length,
        file_offset,
      })),
    ).toEqual([
      {
        name: "weight",
        shape: ["2", "2"],
        byte_length: 16,
        file_offset: fixture.dataOffset,
      },
      {
        name: "quant",
        shape: ["32"],
        byte_length: 18,
        file_offset: fixture.dataOffset + 32,
      },
    ]);
    expect(result.value.totals).toMatchObject({
      parameter_count: "36",
      tensor_bytes: 34,
    });
    const full = await provider.inspect({ path, hash: "full" });
    if (!full.ok) throw full.error;
    expect(full.value.artifact.sha256).toBe(hash(fixture.bytes));
    expect(full.value.artifact.header_sha256).toBe(
      result.value.artifact.header_sha256,
    );
  },
);

it.each([false, true])(
  "preserves GGUF empty extents, quantized empty rows and scalars, endian=%s",
  async (bigEndian) => {
    const declarations = [
      { name: "empty_first", type: 0, shape: [0n, 7n], offset: 0n, bytes: 0 },
      { name: "empty_later", type: 0, shape: [3n, 0n], offset: 0n, bytes: 0 },
      {
        name: "empty_quant_first",
        type: 2,
        shape: [0n, 32n],
        offset: 0n,
        bytes: 0,
      },
      {
        name: "empty_quant_later",
        type: 2,
        shape: [32n, 0n],
        offset: 0n,
        bytes: 0,
      },
      { name: "scalar", type: 0, shape: [], offset: 0n, bytes: 4 },
    ];
    const fixture = ggufFixture({ bigEndian, tensors: declarations });
    const { provider, path } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (!result.ok) throw result.error;
    expect(
      result.value.tensors.map(
        ({ name, shape, byte_length, parameter_count }) => ({
          name,
          shape,
          byte_length,
          parameter_count,
        }),
      ),
    ).toEqual(
      declarations.map(({ name, shape, bytes }) => ({
        name,
        shape: shape.map(String),
        byte_length: bytes,
        parameter_count: name === "scalar" ? "1" : "0",
      })),
    );
    expect(result.value.totals).toMatchObject({
      parameter_count: "1",
      tensor_bytes: 4,
    });
    expect(result.value.artifact.bytes_read).toBe(fixture.header.length);
  },
);

it.each([
  { name: "excess rank", type: 0, shape: [1n, 1n, 1n, 1n, 1n], bytes: 4 },
  { name: "quantized scalar", type: 2, shape: [], bytes: 0 },
])(
  "rejects GGUF $name without relaxing representation constraints",
  async (tensor) => {
    const fixture = ggufFixture({ tensors: [{ ...tensor, offset: 0n }] });
    const { provider, path } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    expect(result.ok).toBe(false);
    if (result.ok) throw new Error("Expected format refusal");
    expect(result.error).toMatchObject({
      constraint: "file_format_rejected",
    });
  },
);

it.each([8, 24, 40])(
  "accepts format-valid GGUF alignment %s and preserves exact offsets",
  async (alignment) => {
    const fixture = ggufFixture({ alignment });
    const { path, provider } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (!result.ok) throw result.error;
    expect(result.value.alignment).toBe(alignment);
    expect(result.value.tensor_data_offset).toBe(fixture.dataOffset);
    expect(
      result.value.tensors.every((row) => row.file_offset % alignment === 0),
    ).toBe(true);
    expect(result.value.tensors[1]?.file_offset).toBe(
      fixture.dataOffset + Math.ceil(16 / alignment) * alignment,
    );
  },
);

it("aligns the GGUF data section after an empty tensor directory", async () => {
  const base = ggufFixture({ tensors: [], metadata: [] });
  const dataOffset = Math.ceil(base.header.length / 32) * 32;
  const bytes = Buffer.concat([
    base.header,
    Buffer.alloc(dataOffset - base.header.length),
  ]);
  const { path, provider } = await providerAt(bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.tensors).toEqual([]);
  expect(result.value.tensor_data_offset).toBe(dataOffset);
  expect(result.value.artifact.header_length).toBe(base.header.length);
});

it("rejects an empty GGUF whose required directory padding is truncated", async () => {
  const fixture = ggufFixture({ tensors: [], metadata: [] });
  const { path, provider } = await providerAt(fixture.header);
  const result = await provider.inspect({ path, hash: "header" });
  expect(result.ok).toBe(false);
});

it.each([0, 1, 2, 4, 12])(
  "rejects GGUF alignment %s outside positive multiples of eight",
  async (alignment) => {
    const base = ggufFixture();
    const fixture = ggufFixture({
      metadata: [
        { key: "general.alignment", type: 4, value: base.u32(alignment) },
      ],
    });
    const { path, provider } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (result.ok) throw new Error("Invalid alignment accepted");
    expect(result.error).toMatchObject({
      _tag: "BinaryTargetError",
      constraint: "file_format_rejected",
    });
  },
);

it.each([
  "General.name",
  "general.naïve",
  "general..name",
  "general/name",
  "general.-name",
  "general.name_",
  "general.__name",
])(
  "rejects invalid GGUF metadata key %s before publishing facts",
  async (key) => {
    const fixture = ggufFixture({
      metadata: [{ key, type: 7, value: Buffer.from([1]) }],
    });
    const { path, provider } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (result.ok) throw new Error("Invalid metadata key accepted");
    expect(result.error).toMatchObject({
      _tag: "BinaryTargetError",
      constraint: "file_format_rejected",
    });
  },
);

it("accepts standardized GGUF metadata keys containing numeric hierarchy components", async () => {
  const base = ggufFixture();
  const fixture = ggufFixture({
    metadata: [
      { key: "general.base_model.0.name", type: 8, value: base.text("parent") },
    ],
  });
  const { path, provider } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.metadata[0]).toMatchObject({
    key: "general.base_model.0.name",
    value: "parent",
  });
});

it.each([99_999_993, 100_000_000])(
  "admits safetensors declared header length %s before its independent representation limit",
  async (length) => {
    const bytes = Buffer.alloc(9);
    bytes.writeBigUInt64LE(BigInt(length));
    bytes[8] = 0x7b;
    const { path, provider } = await providerAt(bytes);
    const file = await open(path, "r+");
    try {
      await file.truncate(8 + length);
    } finally {
      await file.close();
    }
    const result = await provider.inspect({ path, hash: "header" });
    if (result.ok) throw new Error("Representation limit was bypassed");
    expect(result.error).toMatchObject({
      _tag: "AnalysisResourceConstraintError",
      reason: expect.stringContaining("32 MiB"),
    });
  },
);

it("retains the absolute header budget for GGUF without allocating that extent", async () => {
  const { path } = await providerAt(Buffer.alloc(0));
  const file = await open(path, "r+");
  onTestFinished(() => file.close());
  await file.truncate(100_000_009);
  const reader = new ModelHeaderReader(file, 100_000_009);
  expect(() => reader.guarantee(100_000_000)).not.toThrow();
  expect(() => reader.guarantee(100_000_001)).toThrow(ModelFormatFailure);
  try {
    reader.guarantee(100_000_001);
  } catch (cause) {
    expect(cause).toBeInstanceOf(ModelFormatFailure);
    if (!(cause instanceof ModelFormatFailure))
      throw new Error("Expected a model header capacity failure");
    expect(cause.limit).toBe("header-bytes");
    expect(cause.message).toContain("100,000,000 bytes");
    expect(cause.message).not.toContain("Safetensors");
  }
});

it("names exact offset and nesting limits instead of the retained report allowance", async () => {
  const base = ggufFixture({ tensors: [] });
  const oversized = ggufFixture({
    metadata: [
      {
        key: "custom.length",
        type: 8,
        value: base.u64(BigInt(Number.MAX_SAFE_INTEGER) + 1n),
      },
    ],
    tensors: [],
  });
  let nested = Buffer.from([1]);
  for (let frame = 0; frame < 33; frame += 1)
    nested = Buffer.concat([
      base.u32(frame === 0 ? 7 : 9),
      base.u64(1n),
      nested,
    ]);
  const deep = ggufFixture({
    metadata: [{ key: "custom.nested", type: 9, value: nested }],
    tensors: [],
  });
  for (const [label, bytes, allowance] of [
    ["offset", oversized.bytes, String(Number.MAX_SAFE_INTEGER)],
    ["nesting", deep.bytes, "32 array frames"],
  ] as const) {
    const { path, provider } = await providerAt(bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (result.ok) throw new Error(`${label} capacity was accepted`);
    expect(result.error).toMatchObject({
      _tag: "AnalysisResourceConstraintError",
      retryable: false,
      remediationAction: expect.stringContaining(allowance),
    });
    const remediation =
      "remediationAction" in result.error ? result.error.remediationAction : "";
    expect(remediation).not.toContain(String(MODEL_REPORT_BYTES));
    expect(remediation).not.toContain("100,000,000");
    expect(result.error).toMatchObject({
      reportedLimits: expect.not.objectContaining({
        maximum_report_bytes: MODEL_REPORT_BYTES,
      }),
    });
  }
});

it("preserves safetensors scalar, empty and packed tensor declarations and raw string metadata", async () => {
  const fixture = safetensorsFixture();
  const { provider, path } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.artifact.header_sha256).toBe(hash(fixture.header));
  expect(
    result.value.tensors.map((row) => [row.name, row.shape, row.byte_length]),
  ).toEqual([
    ["empty", ["0", "7"], 0],
    ["weight", ["2", "2"], 16],
  ]);
  expect(result.value.metadata.map((row) => [row.key, row.value])).toEqual([
    ["format", "pt"],
    ["license", "Apache-2.0"],
  ]);
  const packed = safetensorsFixture(
    JSON.stringify({
      scalar: { dtype: "F4", shape: [2], data_offsets: [0, 1] },
      empty: { dtype: "F32", shape: [0], data_offsets: [0, 0] },
    }),
    Buffer.alloc(1),
  );
  await writeFile(path, packed.bytes);
  const checked = await provider.inspect({ path, hash: "full" });
  if (!checked.ok) throw checked.error;
  expect(checked.value.totals).toMatchObject({
    parameter_count: "2",
    tensor_bytes: 1,
  });
});

it("returns unknown lengths for unknown GGUF types without inventing storage size", async () => {
  const fixture = ggufFixture({
    tensors: [{ name: "future", type: 999, shape: [1n], offset: 0n, bytes: 1 }],
  });
  const { provider, path } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.tensors[0]).toMatchObject({
    type_id: 999,
    type: "UNKNOWN_999",
    byte_length: null,
    byte_length_state: "unknown",
  });
  expect(result.value.coverage.tensor_ranges).toBe("partial");
  expect(result.value.totals.tensor_bytes).toBeNull();
});

it("keeps unknown safetensors storage validation separate from observed valid byte ranges", async () => {
  const fixture = safetensorsFixture(
    '{"future":{"dtype":"FUTURE4","shape":[2],"data_offsets":[0,4]}}',
    Buffer.alloc(4),
  );
  const { path, provider } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.tensors[0]).toMatchObject({
    byte_length: 4,
    byte_length_state: "declared",
    storage_validation: "unknown",
  });
  expect(result.value.coverage).toMatchObject({
    tensor_ranges: "complete",
    storage_sizes: "partial",
  });
});

it("preserves nonfinite floating metadata and negative zero without JSON coercion", async () => {
  const values = [NaN, Infinity, -Infinity, -0];
  const fixture = ggufFixture({
    metadata: values.map((value, index) => {
      const b = Buffer.alloc(8);
      b.writeDoubleLE(value);
      return { key: `custom.float_${String(index)}`, type: 12, value: b };
    }),
  });
  const { path, provider } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.metadata.map((row) => [row.type, row.value])).toEqual(
    ["NaN", "+Infinity", "-Infinity", "-0"].map((value) => ["FLOAT64", value]),
  );
});

it("validates booleans beyond an array preview without allocating the full value list", async () => {
  const base = ggufFixture();
  const values = Buffer.alloc(10000);
  values[9999] = 2;
  const fixture = ggufFixture({
    metadata: [
      {
        key: "custom.flags",
        type: 9,
        value: Buffer.concat([base.u32(7), base.u64(10000n), values]),
      },
    ],
  });
  const { path, provider } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (result.ok) throw new Error("Invalid omitted boolean was accepted");
  expect(result.error).toMatchObject({
    _tag: "BinaryTargetError",
    constraint: "file_format_rejected",
  });
});

it.each([false, true])(
  "validates omitted UTF-8 strings across chunk boundaries, big endian=%s",
  async (bigEndian) => {
    const base = ggufFixture({ bigEndian });
    const preview = Array.from({ length: 8 }, (_, n) => `\uFEFFtoken-${n} 🦙`);
    const strings = [
      ...preview,
      "",
      "a".repeat(65533) + "€",
      "a".repeat(65535) + "🦙",
      "\0tail",
    ];
    const fixture = ggufFixture({
      bigEndian,
      metadata: [
        {
          key: "custom.strings",
          type: 9,
          value: Buffer.concat([
            base.u32(8),
            base.u64(BigInt(strings.length)),
            ...strings.map(base.text),
          ]),
        },
      ],
    });
    const { path, provider } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (!result.ok) throw result.error;
    expect(result.value.metadata[0]).toMatchObject({
      value: preview,
      element_count: strings.length,
      preview_complete: false,
    });
    expect(result.value.artifact).toMatchObject({
      bytes_read: fixture.header.length,
      header_sha256: createHash("sha256").update(fixture.header).digest("hex"),
    });
    expect(result.value.tensors).toHaveLength(2);
  },
);

it.each([
  ["short", Buffer.from([0xc0, 0x80])],
  [
    "chunk-sized",
    Buffer.concat([Buffer.alloc(65535, 0x61), Buffer.from([0xe2])]),
  ],
  [
    "cross-chunk",
    Buffer.concat([
      Buffer.alloc(65535, 0x61),
      Buffer.from([0xf0, 0x9f, 0x61, 0x80]),
    ]),
  ],
] as const)(
  "rejects malformed omitted %s UTF-8 strings",
  async (_, invalid) => {
    const base = ggufFixture();
    const fixture = ggufFixture({
      metadata: [
        {
          key: "custom.strings",
          type: 9,
          value: Buffer.concat([
            base.u32(8),
            base.u64(9n),
            ...Array.from({ length: 8 }, () => base.text("valid")),
            base.u64(BigInt(invalid.length)),
            invalid,
          ]),
        },
      ],
    });
    const { path, provider } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (result.ok) throw new Error("Invalid omitted UTF-8 string was accepted");
    expect(result.error).toMatchObject({
      _tag: "BinaryTargetError",
      constraint: "file_format_rejected",
      message: expect.stringContaining("GGUF string is not valid UTF-8"),
    });
  },
);

it("chooses model recovery from the producing limit", () => {
  const header = modelCapacityRemediation("header-bytes");
  expect(header).toContain("100,000,000 bytes");
  expect(header).not.toContain("Safetensors");
  expect(header).not.toContain(String(MODEL_REPORT_BYTES));
  const report = modelCapacityRemediation("report-bytes");
  expect(report).toContain(`report allowance is ${String(MODEL_REPORT_BYTES)}`);
  expect(report).not.toContain("representation");
  const representation = modelCapacityRemediation("representation-bytes");
  expect(representation).toContain("representation allowance");
  expect(representation).not.toContain("report allowance");
  const offset = modelCapacityRemediation("exact-offset");
  expect(offset).toContain(String(Number.MAX_SAFE_INTEGER));
  expect(offset).not.toContain(String(MODEL_REPORT_BYTES));
  const nesting = modelCapacityRemediation("metadata-nesting");
  expect(nesting).toContain(`${String(MODEL_METADATA_NESTING)} array frames`);
  expect(nesting).not.toContain(String(MODEL_REPORT_BYTES));
  const jsonNesting = modelCapacityRemediation("json-nesting");
  expect(jsonNesting).toContain(`${String(MODEL_JSON_NESTING)} levels`);
  expect(jsonNesting).not.toContain(String(MODEL_REPORT_BYTES));
  expect(modelCapacityReportedLimits("report-bytes")).toEqual({
    maximum_report_bytes: MODEL_REPORT_BYTES,
  });
  expect(modelCapacityReportedLimits("exact-offset")).toEqual({
    maximum_exact_offset: Number.MAX_SAFE_INTEGER,
  });
  expect(modelCapacityReportedLimits("json-nesting")).toEqual({
    maximum_json_depth: MODEL_JSON_NESTING,
  });
});

it("refuses an oversized declared header before reading or expanding it", async () => {
  const bytes = Buffer.alloc(9);
  bytes.writeBigUInt64LE(100_000_001n);
  bytes[8] = 0x7b;
  const { path, provider } = await providerAt(bytes);
  const file = await open(path, "r+");
  try {
    await file.truncate(100_000_009);
  } finally {
    await file.close();
  }
  const result = await provider.inspect({ path, hash: "header" });
  if (result.ok) throw new Error("Oversized header was accepted");
  expect(result.error).toMatchObject({
    _tag: "AnalysisResourceConstraintError",
    retryable: false,
    reportedLimits: { maximum_header_bytes: 100_000_000 },
    remediationAction: expect.stringContaining("100,000,000"),
  });
  expect(result.error.message).not.toContain("Safetensors");
  expect(
    "remediationAction" in result.error ? result.error.remediationAction : "",
  ).not.toContain("Safetensors");
  expect(
    "reportedLimits" in result.error ? result.error.reportedLimits : {},
  ).not.toHaveProperty("maximum_report_bytes");
});

const malformed = [
  ["truncated GGUF", () => ggufFixture().bytes.subarray(0, 21)],
  [
    "invalid UTF8",
    () => {
      const f = ggufFixture({
        metadata: [
          {
            key: "x",
            type: 8,
            value: Buffer.concat([
              Buffer.from([1, 0, 0, 0, 0, 0, 0, 0]),
              Buffer.from([0xff]),
            ]),
          },
        ],
      });
      return f.bytes;
    },
  ],
  [
    "duplicate GGUF metadata",
    () =>
      ggufFixture({
        metadata: [
          { key: "x", type: 7, value: Buffer.from([0]) },
          { key: "x", type: 7, value: Buffer.from([1]) },
        ],
      }).bytes,
  ],
  [
    "invalid boolean",
    () =>
      ggufFixture({
        metadata: [{ key: "x", type: 7, value: Buffer.from([2]) }],
      }).bytes,
  ],
  [
    "misaligned GGUF tensor",
    () =>
      ggufFixture({
        tensors: [{ name: "x", type: 0, shape: [1n], offset: 1n, bytes: 4 }],
      }).bytes,
  ],
  [
    "invalid quantization row",
    () =>
      ggufFixture({
        tensors: [
          { name: "x", type: 2, shape: [16n, 2n], offset: 0n, bytes: 18 },
        ],
      }).bytes,
  ],
  [
    "overlapping GGUF tensors",
    () =>
      ggufFixture({
        tensors: [
          { name: "a", type: 0, shape: [1n], offset: 0n, bytes: 4 },
          { name: "b", type: 0, shape: [1n], offset: 0n, bytes: 4 },
        ],
      }).bytes,
  ],
  [
    "duplicate JSON keys",
    () =>
      safetensorsFixture(
        '{"x":{"dtype":"F32","dtype":"F16","shape":[2],"data_offsets":[0,4]}}',
        Buffer.alloc(4),
      ).bytes,
  ],
  [
    "safetensors dimension exceeding uint64",
    () =>
      safetensorsFixture(
        '{"x":{"dtype":"F32","shape":[0,18446744073709551616],"data_offsets":[0,0]}}',
        Buffer.alloc(0),
      ).bytes,
  ],
  [
    "overlong safetensors integer",
    () =>
      safetensorsFixture(
        '{"x":{"dtype":"F32","shape":[0,100000000000000000000],"data_offsets":[0,0]}}',
        Buffer.alloc(0),
      ).bytes,
  ],
  [
    "safetensors shape product overflowing before a zero extent",
    () =>
      safetensorsFixture(
        '{"x":{"dtype":"F32","shape":[18446744073709551615,18446744073709551615,0],"data_offsets":[0,0]}}',
        Buffer.alloc(0),
      ).bytes,
  ],
  [
    "safetensors shape mismatch",
    () =>
      safetensorsFixture(
        '{"x":{"dtype":"F32","shape":[3],"data_offsets":[0,16]}}',
      ).bytes,
  ],
  [
    "safetensors gap",
    () =>
      safetensorsFixture(
        '{"x":{"dtype":"F32","shape":[3],"data_offsets":[4,16]}}',
      ).bytes,
  ],
  [
    "safetensors overlap",
    () =>
      safetensorsFixture(
        '{"a":{"dtype":"F32","shape":[4],"data_offsets":[0,16]},"b":{"dtype":"F32","shape":[1],"data_offsets":[0,4]}}',
      ).bytes,
  ],
  ["safetensors trailing data", () => safetensorsFixture("{}").bytes],
  [
    "safetensors non-string metadata",
    () => safetensorsFixture('{"__metadata__":{"x":1}}', Buffer.alloc(0)).bytes,
  ],
  [
    "safetensors decimal metadata container",
    () => safetensorsFixture('{"__metadata__":1.25}', Buffer.alloc(0)).bytes,
  ],
  [
    "safetensors negative metadata container",
    () => safetensorsFixture('{"__metadata__":-1}', Buffer.alloc(0)).bytes,
  ],
  [
    "safetensors overflowing metadata container",
    () =>
      safetensorsFixture(
        '{"__metadata__":18446744073709551616}',
        Buffer.alloc(0),
      ).bytes,
  ],
  [
    "safetensors noninteger shape sentinel",
    () =>
      safetensorsFixture(
        '{"x":{"dtype":"F32","shape":[1.25],"data_offsets":[0,0]}}',
        Buffer.alloc(0),
      ).bytes,
  ],
  [
    "safetensors negative data-offset sentinel",
    () =>
      safetensorsFixture(
        '{"x":{"dtype":"F32","shape":[1],"data_offsets":[-1,4]}}',
        Buffer.alloc(0),
      ).bytes,
  ],
  [
    "safetensors tensor missing a required declaration",
    () =>
      safetensorsFixture('{"x":{"dtype":"F32","shape":[1]}}', Buffer.alloc(0))
        .bytes,
  ],
  [
    "safetensors unpaired surrogate",
    () =>
      safetensorsFixture('{"__metadata__":{"x":"\\ud800"}}', Buffer.alloc(0))
        .bytes,
  ],
] as const;
it.each(malformed)(
  "rejects %s with typed file_format_rejected",
  async (_name, make) => {
    const { provider, path } = await providerAt(make());
    const result = await provider.inspect({ path, hash: "header" });
    if (result.ok) throw new Error("Malformed model was accepted");
    expect(result.error).toMatchObject({
      _tag: "BinaryTargetError",
      constraint: "file_format_rejected",
    });
  },
);

it("accepts null safetensors metadata and ignores tensor extension fields", async () => {
  const fixture = safetensorsFixture(
    '{"__metadata__":null,"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":{"source":"local","flags":[null,false,-1,1.25]}}}',
    Buffer.alloc(4),
  );
  const { provider, path } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.metadata).toEqual([]);
  expect(
    result.value.tensors.map(({ name, type, shape }) => ({
      name,
      type,
      shape,
    })),
  ).toEqual([{ name: "weight", type: "F32", shape: ["1"] }]);
});

it("accepts nested JSON values in ignored Safetensors extension fields", async () => {
  const nested = `${"[".repeat(12)}0${"]".repeat(12)}`;
  const fixture = safetensorsFixture(
    `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,
    Buffer.alloc(4),
  );
  const { provider, path } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  expect(result.ok).toBe(true);
});

it("classifies over-depth Safetensors extension JSON as a resource limit", async () => {
  const nested = `${"[".repeat(MODEL_JSON_NESTING + 1)}0${"]".repeat(MODEL_JSON_NESTING + 1)}`;
  const fixture = safetensorsFixture(
    `{"weight":{"dtype":"F32","shape":[1],"data_offsets":[0,4],"note":${nested}}}`,
    Buffer.alloc(4),
  );
  const { provider, path } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  expect(result.ok).toBe(false);
  if (result.ok) throw new Error("Over-depth JSON was accepted");
  expect(result.error).toMatchObject({
    _tag: "AnalysisResourceConstraintError",
    resource: "memory",
    reason: expect.stringContaining("Safetensors JSON nesting"),
    reportedLimits: { maximum_json_depth: MODEL_JSON_NESTING },
  });
});

it.each([640, 896, 1152, 1408, 66_176])(
  "admits safetensors header length %s whose prefix bytes resemble a pickle protocol marker",
  async (length) => {
    const header = JSON.stringify({
      weight: { dtype: "F32", shape: [1], data_offsets: [0, 4] },
    });
    const fixture = safetensorsFixture(
      header.padEnd(length, " "),
      Buffer.alloc(4),
    );
    const { provider, path } = await providerAt(fixture.bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (!result.ok) throw result.error;
    expect(result.value).toMatchObject({ format: "safetensors" });
    expect(result.value.tensors.map(({ name }) => name)).toEqual(["weight"]);
  },
);

it.each([
  // Protocol 2 pickle of {"a": 1}: PROTO 2, EMPTY_DICT, BINPUT 0, ...
  ["a real pickle", Buffer.from("80027d710058010000006171014b01732e", "hex")],
  // Protocol 2 pickle of {}: six bytes, shorter than a safetensors prefix.
  ["an empty dict", Buffer.from("80027d71002e", "hex")],
  // Protocol 4 FRAME pickle of {"a": 1}.
  [
    "a protocol 4 frame",
    Buffer.from("8004950a000000000000007d948c0161944b01732e", "hex"),
  ],
  // The magic number torch.save writes first in its legacy format.
  [
    "the legacy torch.save magic",
    Buffer.from("80028a0a6cfc9c46f9206aa850192e", "hex"),
  ],
  // Protocol 2 pickle of "x{": its brace lands at offset 8.
  [
    "a brace after an inadmissible length",
    Buffer.from("80025802000000787b71002e", "hex"),
  ],
  [
    "an in-file length without a JSON brace",
    Buffer.concat([
      Buffer.from("8002000000000000", "hex"),
      Buffer.alloc(644, 0x2e),
    ]),
  ],
] as const)(
  "refuses pickle protocol bytes followed by %s",
  async (_case, bytes) => {
    const { provider, path } = await providerAt(bytes);
    const result = await provider.inspect({ path, hash: "header" });
    if (result.ok) throw new Error("Pickle admitted");
    expect(result.error).toMatchObject({
      _tag: "AnalysisUnsupportedTargetError",
      reason: expect.stringContaining("PyTorch pickle"),
    });
  },
);

it("reports a truncated safetensors header whose length resembles a pickle marker as malformed", async () => {
  const fixture = safetensorsFixture(
    JSON.stringify({
      weight: { dtype: "F32", shape: [1], data_offsets: [0, 4] },
    }).padEnd(896, " "),
    Buffer.alloc(4),
  );
  const { provider, path } = await providerAt(fixture.bytes.subarray(0, 500));
  const result = await provider.inspect({ path, hash: "header" });
  if (result.ok) throw new Error("Truncated model was accepted");
  expect(result.error).toMatchObject({
    _tag: "BinaryTargetError",
    constraint: "file_format_rejected",
  });
});

it("bounds expansion before allocation and distinguishes unsupported pickle, version and paths", async () => {
  const fixture = ggufFixture();
  fixture.bytes.writeBigUInt64LE(0xffffffffffffffffn, 8);
  const { provider, path, root } = await providerAt(fixture.bytes);
  const huge = await provider.inspect({ path, hash: "header" });
  if (huge.ok) throw new Error("Huge count admitted");
  expect(huge.error._tag).toBe("AnalysisResourceConstraintError");
  await writeFile(path, Buffer.from([0x80, 4, 0, 0]));
  const pickle = await provider.inspect({ path, hash: "header" });
  if (pickle.ok) throw new Error("Pickle admitted");
  expect(pickle.error._tag).toBe("AnalysisUnsupportedTargetError");
  await writeFile(path, ggufFixture({ version: 999 }).bytes);
  const version = await provider.inspect({ path, hash: "header" });
  if (version.ok) throw new Error("Unknown version admitted");
  expect(version.error._tag).toBe("AnalysisUnsupportedTargetError");
  const alias = join(root, "link");
  await symlink(path, alias);
  const linked = await provider.inspect({ path: alias, hash: "header" });
  expect(linked.ok).toBe(false);
  const service = createModelArtifactService();
  onTestFinished(() => service.close());
  const invalid = await service.inspect({
    path: "relative",
    detail: "complete",
  });
  if (invalid.ok) throw new Error("Relative path admitted");
  expect(invalid.error._tag).toBe("AnalysisInputError");
});

it("reads only the exact header, even when it ends in the middle of a normal buffer", async () => {
  const fixture = safetensorsFixture();
  const { path } = await providerAt(fixture.bytes);
  const file = await open(path, "r");
  onTestFinished(() => file.close());
  const requests: [number, number][] = [];
  const read = file.read.bind(file);
  const boundary = {
    read: async (
      buffer: Buffer,
      offset: number,
      length: number,
      position: number,
    ) => {
      requests.push([position, length]);
      expect(position + length).toBeLessThanOrEqual(fixture.header.length);
      return read(buffer, offset, length, position);
    },
  } as Pick<typeof file, "read">;
  const reader = new ModelHeaderReader(
    boundary as typeof file,
    fixture.bytes.length,
  );
  await readSafetensorsHeader(reader, await reader.take(4));
  expect(await reader.finish(false)).toMatchObject({
    bytes_read: fixture.header.length,
    header_sha256: hash(fixture.header),
  });
  expect(requests.reduce((n, [, length]) => n + length, 0)).toBe(
    fixture.header.length,
  );
});

it.each(["header", "full"] as const)(
  "classifies real truncation after admission during %s reading as an identity change",
  async (mode) => {
    const fixture = ggufFixture();
    const { path } = await providerAt(fixture.bytes);
    const file = await open(path, "r");
    onTestFinished(() => file.close());
    const admitted = await file.stat();
    const reader = new ModelHeaderReader(file, admitted.size);
    if (mode === "full") await reader.take(fixture.header.length);
    await truncate(path, mode === "header" ? 0 : fixture.header.length);
    const reading = mode === "header" ? reader.take(4) : reader.finish(true);
    await expect(reading).rejects.toMatchObject({ reason: "integrity" });
  },
);

it("keeps GGUF read-ahead inside the directory and hashes precisely that prefix", async () => {
  const fixture = ggufFixture();
  const { path } = await providerAt(fixture.bytes);
  const file = await open(path, "r");
  onTestFinished(() => file.close());
  const read = file.read.bind(file);
  let bytesRead = 0;
  const boundary = {
    read: async (
      buffer: Buffer,
      offset: number,
      length: number,
      position: number,
    ) => {
      expect(position + length).toBeLessThanOrEqual(fixture.header.length);
      const result = await read(buffer, offset, length, position);
      bytesRead += result.bytesRead;
      return result;
    },
  } as Pick<typeof file, "read">;
  const reader = new ModelHeaderReader(
    boundary as typeof file,
    fixture.bytes.length,
  );
  await reader.take(4);
  await new GgufHeader(reader).read();
  await reader.finish(false);
  expect(bytesRead).toBe(fixture.header.length);
});

it("inspects a 1 GiB sparse safetensors declaration using only its small header", async () => {
  const dataBytes = 1024 ** 3;
  const fixture = safetensorsFixture(
    JSON.stringify({
      weight: { dtype: "U8", shape: [dataBytes], data_offsets: [0, dataBytes] },
    }),
    Buffer.alloc(0),
  );
  const { path, provider } = await providerAt(fixture.header);
  const file = await open(path, "r+");
  try {
    await file.truncate(fixture.header.length + dataBytes);
  } finally {
    await file.close();
  }
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.artifact.bytes_read).toBe(fixture.header.length);
  expect(result.value.tensors[0]).toMatchObject({
    byte_length: dataBytes,
    parameter_count: String(dataBytes),
  });
});

it("supports true scalar shapes and prototype-named metadata without changing their spelling", async () => {
  const fixture = safetensorsFixture(
    '{"__metadata__":{"__proto__":"raw","constructor":"kept"},"scalar":{"dtype":"F32","shape":[],"data_offsets":[0,4]}}',
    Buffer.alloc(4),
  );
  const { path, provider } = await providerAt(fixture.bytes);
  const result = await provider.inspect({ path, hash: "header" });
  if (!result.ok) throw result.error;
  expect(result.value.tensors[0]).toMatchObject({
    shape: [],
    parameter_count: "1",
    byte_length: 4,
  });
  expect(result.value.metadata.map(({ key, value }) => [key, value])).toEqual([
    ["__proto__", "raw"],
    ["constructor", "kept"],
  ]);
});

it("yields while validating a large metadata array, preserves cancellation and releases admission", async () => {
  const f = ggufFixture();
  const count = 1_000_000;
  const fixture = ggufFixture({
    metadata: [
      {
        key: "custom.bools",
        type: 9,
        value: Buffer.concat([
          f.u32(7),
          f.u64(BigInt(count)),
          Buffer.alloc(count),
        ]),
      },
    ],
  });
  const { path, provider } = await providerAt(fixture.bytes);
  const controller = new AbortController();
  const reason = new Error("stop large array");
  const pending = provider.inspect(
    { path, hash: "header" },
    { signal: controller.signal },
  );
  setTimeout(() => controller.abort(reason), 0);
  const result = await pending;
  if (result.ok) throw new Error("Cancellation ignored");
  expect(result.error).toMatchObject({
    _tag: "AnalysisCancelledError",
    cause: reason,
  });
  await writeFile(path, ggufFixture().bytes);
  const recovered = await provider.inspect({ path, hash: "header" });
  if (!recovered.ok) throw recovered.error;
  expect(recovered.value.tensors).toHaveLength(2);
});

it("retains explicit scoped identity through shared service and preserves caller abort reasons", async () => {
  const fixture = ggufFixture();
  const { path } = await providerAt(fixture.bytes);
  const service = createModelArtifactService();
  onTestFinished(() => service.close());
  const result = await service.inspect({ path, detail: "complete" });
  if (!result.ok) throw result.error;
  expect(result.value.subject).toBeNull();
  expect(
    modelArtifactSchema.parse(result.value.normalized_result).artifact.sha256,
  ).toBeNull();
  const full = await service.inspect({
    path,
    hash: "full",
    detail: "complete",
  });
  if (!full.ok) throw full.error;
  expect(full.value.subject?.digest.sha256).toBe(hash(fixture.bytes));
  const controller = new AbortController();
  const reason = new Error("selected caller stop");
  controller.abort(reason);
  const stopped = await service.inspect(
    { path, detail: "complete" },
    { signal: controller.signal },
  );
  if (stopped.ok) throw new Error("Cancellation ignored");
  expect(stopped.error.cause).toBe(reason);
});