Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(artifacts): admit bounded reads and preserve source identity
morluto6d4cefc4f74e
diff --git a/src/application/AppBundleExecutable.ts b/src/application/AppBundleExecutable.tsindex 7df3ca03a..0ae1c073e 100644--- a/src/application/AppBundleExecutable.ts+++ b/src/application/AppBundleExecutable.ts@@ -1,5 +1,5 @@import { execFile } from "node:child_process";-import { lstat, readdir, readFile, realpath, stat } from "node:fs/promises";+import { lstat, opendir, readFile, realpath, stat } from "node:fs/promises";import { join } from "node:path";import { promisify } from "node:util";@@ -156,13 +156,14 @@ const wrappedBundle = async (bundle: string): Promise<string | undefined> => {if (isAbsence(cause)) return undefined;throw cause;}- const apps = (await readdir(wrapper, { withFileTypes: true })).filter(- (entry) => entry.isDirectory() && entry.name.toLowerCase().endsWith(".app"),- );- const [app] = apps;- return apps.length === 1 && app !== undefined- ? join(wrapper, app.name)- : undefined;+ let app: string | undefined;+ for await (const entry of await opendir(wrapper)) {+ if (!entry.isDirectory() || !entry.name.toLowerCase().endsWith(".app"))+ continue;+ if (app !== undefined) return undefined;+ app = join(wrapper, entry.name);+ }+ return app;};const resolveLayoutExecutable = async (diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.tsindex 3163d17b3..8b532b0d5 100644--- a/src/application/BinaryTargetResolver.ts+++ b/src/application/BinaryTargetResolver.ts@@ -1,4 +1,4 @@-import { constants } from "node:fs";+import { constants, type Stats } from "node:fs";import { createHash } from "node:crypto";import {access,@@ -248,7 +248,7 @@ const isArchiveFormat = (const sha256Handle = async (handle: FileHandle,path: string,- initial: Awaited<ReturnType<FileHandle["stat"]>>,+ initial: Stats,signal?: AbortSignal,): Promise<string> => {const hash = createHash("sha256");@@ -256,11 +256,18 @@ const sha256Handle = async (let position = 0;while (true) {throwIfTargetResolutionCancelled(signal);- const observed = await handle.read(chunk, 0, chunk.length, position);+ const observed = await handle.read(+ chunk,+ 0,+ Math.min(chunk.length, initial.size - position + 1),+ position,+ );throwIfTargetResolutionCancelled(signal);if (observed.bytesRead === 0) break;hash.update(chunk.subarray(0, observed.bytesRead));position += observed.bytesRead;+ if (position > initial.size)+ throw new BinaryTargetError(path, "target grew while being read");}throwIfTargetResolutionCancelled(signal);const [opened, currentPath] = await Promise.all([handle.stat(), lstat(path)]);diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.tsindex 0ccca760a..916d4fd61 100644--- a/src/application/JsonFiles.ts+++ b/src/application/JsonFiles.ts@@ -42,7 +42,7 @@ export const readJsonFile = async (if (!decoded.ok) {return err(new EvidenceFileError("read", "invalid-json", {- cause: decoded.cause,+ cause: decoded.cause ?? new SyntaxError(decoded.error),path: requestedPath,}),);diff --git a/src/artifacts/AsarArtifactReader.ts b/src/artifacts/AsarArtifactReader.tsindex bedd9a5f2..80e9fec0f 100644--- a/src/artifacts/AsarArtifactReader.ts+++ b/src/artifacts/AsarArtifactReader.ts@@ -27,6 +27,7 @@ import {type ArtifactReader,} from "./ArtifactReader.js";import { closeAsarHandle, readValidatedAsarEntry } from "./AsarEntryStream.js";+import { admitAsarHeader } from "./AsarHeader.js";/*** Official Electron ASAR adapter with range-streamed member reads.@@ -57,6 +58,17 @@ export class AsarArtifactReader implements ArtifactReader {const snapshotPath = this.#snapshotPath();this.#resetArchiveState();this.#entries.clear();+ const headerHandle = await open(+ snapshotPath,+ constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,+ );+ try {+ const admitted = await admitAsarHeader(headerHandle);+ this.#archiveSize = admitted.archiveBytes;+ } finally {+ await headerHandle.close();+ }+ abortIfNeeded(signal);paths = listPackage(snapshotPath, { isPack: false }).sort((left, right) =>left.localeCompare(right, "en"),);diff --git a/src/artifacts/AsarHeader.ts b/src/artifacts/AsarHeader.tsnew file mode 100644index 000000000..7c17f1cc7--- /dev/null+++ b/src/artifacts/AsarHeader.ts@@ -0,0 +1,48 @@+import type { FileHandle } from "node:fs/promises";++import { ArtifactReaderFailure } from "./ArtifactReader.js";++/** Shared resource budget for the eagerly decoded ASAR header, not member bytes. */+export const MAX_ASAR_HEADER_BYTES = 16 * 1024 * 1024;++/** Admit both Chromium pickle extents before an upstream decoder allocates them. */+export const admitAsarHeader = async (+ handle: FileHandle,+): Promise<{+ readonly archiveBytes: number;+ readonly headerBytes: number;+ readonly jsonBytes: number;+}> => {+ const observed = await handle.stat();+ if (!observed.isFile())+ throw new ArtifactReaderFailure(+ "format",+ "ASAR container is not a regular file",+ );+ const prefix = Buffer.alloc(16);+ const read = await handle.read(prefix, 0, prefix.length, 0);+ if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4)+ throw new ArtifactReaderFailure(+ "format",+ "Truncated or invalid ASAR size pickle",+ );+ const headerBytes = prefix.readUInt32LE(4);+ const payloadBytes = prefix.readUInt32LE(8);+ const jsonBytes = prefix.readUInt32LE(12);+ if (+ headerBytes < 8 ||+ headerBytes > observed.size - 8 ||+ payloadBytes !== headerBytes - 4 ||+ jsonBytes > payloadBytes - 4+ )+ throw new ArtifactReaderFailure(+ "format",+ "ASAR header leaves its container or has invalid pickle extents",+ );+ if (headerBytes > MAX_ASAR_HEADER_BYTES)+ throw new ArtifactReaderFailure(+ "limit",+ `ASAR header exceeds the ${String(MAX_ASAR_HEADER_BYTES)} byte decode budget`,+ );+ return { archiveBytes: observed.size, headerBytes, jsonBytes };+};diff --git a/src/artifacts/DirectoryArtifactReader.ts b/src/artifacts/DirectoryArtifactReader.tsindex a9e339be5..123a1cb09 100644--- a/src/artifacts/DirectoryArtifactReader.ts+++ b/src/artifacts/DirectoryArtifactReader.ts@@ -3,15 +3,13 @@ import { lstat, open, opendir, realpath } from "node:fs/promises";import { isAbsolute, join, relative, sep } from "node:path";import type { Readable } from "node:stream";-import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";-import {ArtifactReaderFailure,type ArtifactEntry,type ArtifactReader,} from "./ArtifactReader.js";-/** Deterministic, non-symlink-following directory reader. */+/** Non-symlink-following directory reader. */export class DirectoryArtifactReader implements ArtifactReader {readonly format = "directory" as const;readonly #rootPromise: Promise<string>;@@ -38,13 +36,52 @@ export class DirectoryArtifactReader implements ArtifactReader {await withDirectoryIoContext("inspect directory", directory, () =>assertContainedDirectory(root, directory),);- const children: string[] = [];const handle = await opendir(directory).catch((cause: unknown) => {throw directoryIoFailure("open directory", directory, cause);});+ const directories: string[] = [];try {- for await (const child of handle) children.push(child.name);+ for await (const child of handle) {+ abortIfNeeded(signal);+ const absolute = join(directory, child.name);+ const metadata = await lstat(absolute).catch((cause: unknown) => {+ throw directoryIoFailure("inspect entry", absolute, cause);+ });+ const path = relative(root, absolute).split(sep).join("/");+ const kind = metadata.isSymbolicLink()+ ? "symlink"+ : metadata.isDirectory()+ ? "directory"+ : metadata.isFile()+ ? "file"+ : undefined;+ if (kind === undefined) continue;+ yield {+ path,+ kind,+ declaredSize: kind === "file" ? metadata.size : null,+ compressedSize: null,+ executable: (metadata.mode & 0o111) !== 0,+ encrypted: false,+ byteOffset: null,+ declaredSha256: null,+ unpacked: false,+ limitations:+ kind === "symlink"+ ? ["Symlink target was not followed or disclosed."]+ : [],+ adapterKey: absolute,+ ...(kind === "file"+ ? {+ sourceIdentity: { device: metadata.dev, inode: metadata.ino },+ }+ : {}),+ };+ if (kind === "directory" && (descendInto?.(path) ?? true))+ directories.push(absolute);+ }} catch (cause: unknown) {+ if (cause instanceof ArtifactReaderFailure) throw cause;throw directoryIoFailure("read directory", directory, cause);} finally {// best-effort cleanup: directory-handle close must not mask traversal.@@ -53,45 +90,6 @@ export class DirectoryArtifactReader implements ArtifactReader {await withDirectoryIoContext("inspect directory", directory, () =>assertContainedDirectory(root, directory),);- children.sort(compareUnicodeCodePoints);- const directories: string[] = [];- for (const name of children) {- abortIfNeeded(signal);- const absolute = join(directory, name);- const metadata = await lstat(absolute).catch((cause: unknown) => {- throw directoryIoFailure("inspect entry", absolute, cause);- });- const path = relative(root, absolute).split(sep).join("/");- const kind = metadata.isSymbolicLink()- ? "symlink"- : metadata.isDirectory()- ? "directory"- : metadata.isFile()- ? "file"- : undefined;- if (kind === undefined) continue;- if (kind === "directory" && (descendInto?.(path) ?? true))- directories.push(absolute);- yield {- path,- kind,- declaredSize: kind === "file" ? metadata.size : null,- compressedSize: null,- executable: (metadata.mode & 0o111) !== 0,- encrypted: false,- byteOffset: null,- declaredSha256: null,- unpacked: false,- limitations:- kind === "symlink"- ? ["Symlink target was not followed or disclosed."]- : [],- adapterKey: absolute,- ...(kind === "file"- ? { sourceIdentity: { device: metadata.dev, inode: metadata.ino } }- : {}),- };- }directories.reverse();pending.push(...directories);}diff --git a/src/artifacts/SafeOutputTree.ts b/src/artifacts/SafeOutputTree.tsindex 05ce0ac68..d9610192e 100644--- a/src/artifacts/SafeOutputTree.ts+++ b/src/artifacts/SafeOutputTree.ts@@ -105,7 +105,26 @@ export class SafeOutputTree {}return new SafeOutputTree(canonicalOutput, platform);} catch (cause: unknown) {- await rm(canonicalOutput, { recursive: true, force: true });+ let removalFailure: unknown;+ try {+ await rm(canonicalOutput, { recursive: true, force: true });+ } catch (cleanupCause: unknown) {+ removalFailure = cleanupCause;+ }+ let absent = false;+ try {+ absent = await isAbsent(canonicalOutput);+ } catch (cleanupCause: unknown) {+ removalFailure ??= cleanupCause;+ }+ if (!absent)+ throw ArtifactReaderFailure.withCleanup(cause, {+ reason:+ removalFailure instanceof Error+ ? removalFailure.message+ : "Extraction output root remains after setup failure",+ resources: [canonicalOutput],+ });throw cause;}}@@ -122,56 +141,84 @@ export class SafeOutputTree {async write(relativePath: string,source: Readable,- expectedSha256: string,+ expected: { readonly sha256: string; readonly bytes: number },signal?: AbortSignal,): Promise<SafeOutputFile> {- this.#assertWritable();- const path = normalizeArtifactPath(relativePath);- this.#registry.add(path, "file");- const destination = await this.#prepareParent(path);- const handle = await open(- destination,- constants.O_CREAT |- constants.O_EXCL |- constants.O_WRONLY |- constants.O_NOFOLLOW,- 0o600,- ).catch(async (cause: unknown) => {- await throwIfDestinationCaseCollision(dirname(destination), path, cause);- throw new ArtifactReaderFailure(- "path",- `Could not exclusively create extraction path: ${path}`,- { cause },- );- });- const hash = createHash("sha256");- let bytes = 0;try {- for await (const raw of source) {- abortIfNeeded(signal);- const chunk = streamChunkToBuffer(raw);- bytes += chunk.length;- hash.update(chunk);- await writeAll(handle, chunk);- }- const sha256 = hash.digest("hex");- if (sha256 !== expectedSha256)+ this.#assertWritable();+ if (!Number.isSafeInteger(expected.bytes) || expected.bytes < 0)throw new ArtifactReaderFailure(- "integrity",- `Extracted content disagrees with inventory: ${path}`,+ "format",+ `Invalid expected extraction size for ${relativePath}`,+ );+ const path = normalizeArtifactPath(relativePath);+ this.#registry.add(path, "file");+ const destination = await this.#prepareParent(path);+ const handle = await open(+ destination,+ constants.O_CREAT |+ constants.O_EXCL |+ constants.O_WRONLY |+ constants.O_NOFOLLOW,+ 0o600,+ ).catch(async (cause: unknown) => {+ await throwIfDestinationCaseCollision(+ dirname(destination),+ path,+ cause,);- await handle.sync();- await handle.close();- const readback = await hashFile(destination, bytes, signal);- if (readback.sha256 !== sha256 || readback.bytes !== bytes)throw new ArtifactReaderFailure(- "integrity",- `Durable readback verification failed: ${path}`,+ "path",+ `Could not exclusively create extraction path: ${path}`,+ { cause },);- return { relativePath: path, sha256, bytesWritten: bytes };+ });+ const hash = createHash("sha256");+ let bytes = 0;+ try {+ for await (const raw of source) {+ abortIfNeeded(signal);+ const chunk = streamChunkToBuffer(raw);+ if (chunk.length > expected.bytes - bytes)+ throw new ArtifactReaderFailure(+ "integrity",+ `Extracted content exceeds the inventoried size: ${path}`,+ );+ bytes += chunk.length;+ hash.update(chunk);+ await writeAll(handle, chunk);+ }+ if (bytes !== expected.bytes)+ throw new ArtifactReaderFailure(+ "integrity",+ `Extracted content size disagrees with inventory: ${path}`,+ );+ const sha256 = hash.digest("hex");+ if (sha256 !== expected.sha256)+ throw new ArtifactReaderFailure(+ "integrity",+ `Extracted content disagrees with inventory: ${path}`,+ );+ await handle.sync();+ await handle.close();+ const readback = await hashFile(destination, bytes, signal);+ if (readback.sha256 !== sha256 || readback.bytes !== bytes)+ throw new ArtifactReaderFailure(+ "integrity",+ `Durable readback verification failed: ${path}`,+ );+ return { relativePath: path, sha256, bytesWritten: bytes };+ } catch (cause: unknown) {+ // best-effort cleanup: file-handle close must not mask the write failure.+ await handle.close().catch(() => undefined);+ throw cause;+ }} catch (cause: unknown) {- // best-effort cleanup: file-handle close must not mask the write failure.- await handle.close().catch(() => undefined);+ try {+ source.destroy();+ } catch {+ // Preserve the write refusal or failure as the caller-visible error.+ }throw cause;}}diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.tsindex 3c7a92ea1..17637ca45 100644--- a/src/artifacts/ZipArtifactReader.ts+++ b/src/artifacts/ZipArtifactReader.ts@@ -1,4 +1,4 @@-import { open, type FileHandle } from "node:fs/promises";+import type { FileHandle } from "node:fs/promises";import { PassThrough, type Readable } from "node:stream";import { once } from "node:events";@@ -10,20 +10,57 @@ import {type ArtifactReader,} from "./ArtifactReader.js";import type { ZipPackageFormat } from "../domain/zipPackageFormat.js";+import {+ NonRegularFileReadError,+ openRegularFile,+ sameRegularFileState,+ type StableRegularFileDescriptor,+} from "../filesystem/RegularFile.js";class NodeFileReader extends Reader<string> {#handle: FileHandle | undefined;+ #initPromise: Promise<void> | undefined;+ readonly #ownsHandle: boolean;constructor(private readonly path: string,private readonly maximumReadBytes?: number,+ private readonly admitted?: StableRegularFileDescriptor,) {super(path);+ this.#ownsHandle = admitted === undefined;+ }++ override init(): Promise<void> {+ this.#initPromise ??= this.#initialize();+ return this.#initPromise;}- override async init(): Promise<void> {- this.#handle = await open(this.path, "r");- this.size = (await this.#handle.stat()).size;+ async #initialize(): Promise<void> {+ try {+ this.#handle =+ this.admitted?.handle ??+ (await openRegularFile(this.path, { symlinks: "reject" }));+ } catch (cause: unknown) {+ if (cause instanceof NonRegularFileReadError)+ throw new ArtifactReaderFailure(+ "format",+ `ZIP source is not a regular file: ${this.path}`,+ { cause },+ );+ throw cause;+ }+ const metadata = await this.#handle.stat();+ if (+ !metadata.isFile() ||+ (this.admitted !== undefined &&+ !sameRegularFileState(this.admitted.initial, metadata))+ )+ throw new ArtifactReaderFailure(+ "integrity",+ `ZIP source identity changed before reading: ${this.path}`,+ );+ this.size = metadata.size;}override async readUint8Array(@@ -47,7 +84,10 @@ class NodeFileReader extends Reader<string> {}async closeHandle(): Promise<void> {- await this.#handle?.close();+ if (!this.#ownsHandle) return;+ await this.#initPromise?.catch(() => undefined);+ if (this.#handle === undefined) return;+ await this.#handle.close();this.#handle = undefined;}}@@ -64,9 +104,10 @@ export class ZipArtifactReader implements ArtifactReader {path: string,format: ZipPackageFormat,maximumMetadataReadBytes?: number,+ admitted?: StableRegularFileDescriptor,) {this.format = format;- this.#source = new NodeFileReader(path, maximumMetadataReadBytes);+ this.#source = new NodeFileReader(path, maximumMetadataReadBytes, admitted);this.#reader = new ZipReader(this.#source, {checkSignature: true,checkOverlappingEntry: true,diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.tsindex f47c5c56c..e15a81b44 100644--- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts+++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts@@ -25,6 +25,7 @@ import {type NibHierarchyNode,} from "./NibViewHierarchy.js";import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";+import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js";@@ -49,7 +50,6 @@ export const analyzeInterfaceBuilderBundle = async (input: {const prototypeKeyOmissions = new Map<string, number>();let omitted = 0;let attempted = 0;- let candidates = 0;let aggregateInputBytes = 0;const decodeBudget = new InterfaceBuilderDecodeBudget(MAX_AGGREGATE_DECODE_BYTES,@@ -60,15 +60,16 @@ export const analyzeInterfaceBuilderBundle = async (input: {else unreportedOmittedArchives += 1;};try {- for await (const entry of reader.entries(input.signal)) {- if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path))- continue;- candidates += 1;- if (candidates > limits.max_documents) {- omitted += 1;- recordOmittedArchive(entry.path, "max_documents");- continue;- }+ const selection = await selectInterfaceBuilderCandidates(+ reader,+ input.signal,+ limits.max_documents,+ );+ omitted = selection.omitted;+ for (const entry of selection.omittedSamples)+ recordOmittedArchive(entry.path, "max_documents");+ unreportedOmittedArchives += selection.unreportedOmitted;+ for (const entry of selection.selected) {try {const remainingInputBytes =MAX_AGGREGATE_INPUT_BYTES - aggregateInputBytes;@@ -167,6 +168,60 @@ export const analyzeInterfaceBuilderBundle = async (input: {});};+/** Select lexically earliest document paths independent of host traversal order. */+const selectInterfaceBuilderCandidates = async (+ reader: DirectoryArtifactReader,+ signal: AbortSignal | undefined,+ maximumDocuments: number,+): Promise<{+ readonly selected: readonly ArtifactEntry[];+ readonly omittedSamples: readonly ArtifactEntry[];+ readonly omitted: number;+ readonly unreportedOmitted: number;+}> => {+ const capacity = maximumDocuments + MAX_REPORTED_OMITTED_ARCHIVES;+ const candidates: ArtifactEntry[] = [];+ let total = 0;+ for await (const entry of reader.entries(signal)) {+ if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path))+ continue;+ total += 1;+ retainCandidate(candidates, entry, capacity);+ }+ const omittedSamples = candidates.slice(maximumDocuments, capacity);+ return {+ selected: candidates.slice(0, maximumDocuments),+ omittedSamples,+ omitted: Math.max(0, total - maximumDocuments),+ unreportedOmitted: Math.max(+ 0,+ total - maximumDocuments - omittedSamples.length,+ ),+ };+};++/** Keep the earliest bounded path set; directory iteration order is host-defined. */+const retainCandidate = (+ candidates: ArtifactEntry[],+ entry: ArtifactEntry,+ maximum: number,+): void => {+ let low = 0;+ let high = candidates.length;+ while (low < high) {+ const middle = Math.floor((low + high) / 2);+ const existing = candidates[middle];+ if (+ existing !== undefined &&+ compareUnicodeCodePoints(existing.path, entry.path) < 0+ )+ low = middle + 1;+ else high = middle;+ }+ candidates.splice(low, 0, entry);+ if (candidates.length > maximum) candidates.pop();+};+const finalizeHierarchyCoverage = (result: ReturnType<typeof buildInterfaceBuilderAnalysis>,counts: {diff --git a/src/artifacts/apple/KeyedArchiveReader.ts b/src/artifacts/apple/KeyedArchiveReader.tsindex ec50ab4cf..3a24f0962 100644--- a/src/artifacts/apple/KeyedArchiveReader.ts+++ b/src/artifacts/apple/KeyedArchiveReader.ts@@ -147,29 +147,28 @@ export const inspectBundleKeyedArchive = async (input: {try {// Match inventory spelling while retaining the raw path for I/O and evidence.const wanted = selected.path.normalize("NFC");- const matches: ArtifactEntry[] = [];- for await (const entry of reader.entries(input.signal, (directory) =>+ let entry: ArtifactEntry | undefined;+ for await (const candidate of reader.entries(input.signal, (directory) =>wanted.startsWith(`${directory.normalize("NFC")}/`),)) {- if (entry.path.normalize("NFC") !== wanted) continue;- if (entry.kind !== "file")+ if (candidate.path.normalize("NFC") !== wanted) continue;+ if (candidate.kind !== "file")throw archivePathError("invalid_value",- `Archive path selects a ${entry.kind}, not a regular file: ${selected.path}`,+ `Archive path selects a ${candidate.kind}, not a regular file: ${selected.path}`,);- if ((entry.declaredSize ?? 0) > MAX_BYTES)+ if ((candidate.declaredSize ?? 0) > MAX_BYTES)throw new ArtifactReaderFailure("limit","Keyed archive exceeds 64 MiB",);- matches.push(entry);+ if (entry !== undefined)+ throw archivePathError(+ "invalid_value",+ `Archive path matches multiple Unicode-equivalent bundle entries; selection is ambiguous: ${selected.path}`,+ );+ entry = candidate;}- if (matches.length > 1)- throw archivePathError(- "invalid_value",- `Archive path matches ${String(matches.length)} Unicode-equivalent bundle entries; selection is ambiguous: ${selected.path}`,- );- const entry = matches[0];if (entry === undefined)throw archivePathError("invalid_value",diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.tsindex 5bf6e94cf..5fcfba720 100644--- a/src/artifacts/extraction/ArtifactExtraction.ts+++ b/src/artifacts/extraction/ArtifactExtraction.ts@@ -215,7 +215,10 @@ const materializeSelection = async ({const written = await output.write(path,stream,- selectedItem.node.sha256,+ {+ sha256: selectedItem.node.sha256,+ bytes: selectedItem.node.size,+ },signal,);extracted.push({diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.tsindex fe29c6057..6e69c64c8 100644--- a/src/artifacts/inventory/classify.ts+++ b/src/artifacts/inventory/classify.ts@@ -5,6 +5,7 @@ import {} from "../../filesystem/RegularFile.js";import type { FileHandle } from "node:fs/promises";import type { Stats } from "node:fs";+import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js";@@ -17,18 +18,46 @@ import { ArtifactReaderFailure } from "../ArtifactReader.js";import type { HashResult } from "../ArtifactHash.js";import { hashStableRootArtifactHandle } from "./hashStableRootArtifact.js";+interface RootClassification {+ readonly format: ArtifactOccurrence["artifact_format"];+ readonly digest: HashResult | null;+}++type RootInventoryClassification = RootClassification & {+ readonly zipSource: StableRegularFileDescriptor | undefined;+};+/** Classify and hash one file root through the same stable open descriptor. */export const classifyAndHashRoot = async (path: string,directory: boolean,expectedMetadata: Stats,signal?: AbortSignal,-): Promise<{- readonly format: ArtifactOccurrence["artifact_format"];- readonly digest: HashResult | null;-}> => {- if (directory) return { format: "directory", digest: null };+): Promise<RootClassification> => {+ const result = await classifyAndHashRootForInventory(+ path,+ directory,+ expectedMetadata,+ signal,+ );+ try {+ return { format: result.format, digest: result.digest };+ } finally {+ await result.zipSource?.handle.close();+ }+};++/** Classify a root and retain its admitted ZIP descriptor for child inventory. */+export const classifyAndHashRootForInventory = async (+ path: string,+ directory: boolean,+ expectedMetadata: Stats,+ signal?: AbortSignal,+): Promise<RootInventoryClassification> => {+ if (directory)+ return { format: "directory", digest: null, zipSource: undefined };const handle = await openRootFile(path, signal);+ let transferHandle = false;try {const initial = await handle.stat();if (!sameRegularFileState(expectedMetadata, initial))@@ -43,12 +72,25 @@ export const classifyAndHashRoot = async (initial,signal,);- return { format, digest };+ const zipSource = isZipFormat(format);+ if (zipSource) transferHandle = true;+ return {+ format,+ digest,+ zipSource: zipSource ? { handle, initial } : undefined,+ };} finally {- await handle.close();+ if (!transferHandle) await handle.close();}};+const isZipFormat = (format: ArtifactOccurrence["artifact_format"]): boolean =>+ format === "zip" ||+ format === "ipa" ||+ format === "apk" ||+ format === "msix" ||+ format === "appx";+const openRootFile = async (path: string,signal?: AbortSignal,diff --git a/src/artifacts/inventory/hashStableRootArtifact.ts b/src/artifacts/inventory/hashStableRootArtifact.tsindex e441062d3..cd3cc302c 100644--- a/src/artifacts/inventory/hashStableRootArtifact.ts+++ b/src/artifacts/inventory/hashStableRootArtifact.ts@@ -1,4 +1,5 @@import { lstat } from "node:fs/promises";+import type { Stats } from "node:fs";import type { FileHandle } from "node:fs/promises";import { ArtifactReaderFailure } from "../ArtifactReader.js";@@ -45,11 +46,13 @@ export const hashStableRootArtifact = async (export const hashStableRootArtifactHandle = async (path: string,handle: FileHandle,- initial: Awaited<ReturnType<FileHandle["stat"]>>,+ initial: Stats,signal?: AbortSignal,): Promise<HashResult> => {const digest = await hashReadable(- handle.createReadStream({ autoClose: false }),+ // Read at most the admitted extent plus one byte to detect growth without+ // following an actively appended file indefinitely.+ handle.createReadStream({ autoClose: false, start: 0, end: initial.size }),signal,);abortIfNeeded(signal);diff --git a/src/artifacts/inventory/reader.ts b/src/artifacts/inventory/reader.tsindex 692431a00..21ad5adea 100644--- a/src/artifacts/inventory/reader.ts+++ b/src/artifacts/inventory/reader.ts@@ -6,12 +6,14 @@ import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";import { ZipArtifactReader } from "../ZipArtifactReader.js";import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "../../domain/zipPackageFormat.js";+import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";export const createReader = async (path: string,format: ArtifactOccurrence["artifact_format"],environment: Readonly<NodeJS.ProcessEnv>,signal?: AbortSignal,+ zipSource?: StableRegularFileDescriptor,): Promise<ArtifactReader | undefined> => {switch (format) {case "directory":@@ -21,7 +23,7 @@ export const createReader = async (case "apk":case "msix":case "appx":- return new ZipArtifactReader(path, format);+ return new ZipArtifactReader(path, format, undefined, zipSource);case "asar":return new AsarArtifactReader(path);case "mach-o-universal":diff --git a/src/artifacts/inventory/scanCanonical.ts b/src/artifacts/inventory/scanCanonical.tsindex 0719e58f4..9ae0db083 100644--- a/src/artifacts/inventory/scanCanonical.ts+++ b/src/artifacts/inventory/scanCanonical.ts@@ -5,6 +5,7 @@ import { AsarArtifactReader } from "../AsarArtifactReader.js";import { lstat } from "node:fs/promises";import type { Stats } from "node:fs";+import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";import {ArtifactReaderFailure,@@ -31,9 +32,12 @@ import {artifactGraphDigest,artifactManifestId,} from "../../domain/artifactIdentity.js";-import { classifyAndHashRoot } from "./classify.js";+import { classifyAndHashRootForInventory } from "./classify.js";import { type HashResult } from "../ArtifactHash.js";-import { hashStableRootArtifact } from "./hashStableRootArtifact.js";+import {+ hashStableRootArtifact,+ hashStableRootArtifactHandle,+} from "./hashStableRootArtifact.js";import { createReader, inventoryLimitations } from "./reader.js";import {scanReader,@@ -51,18 +55,29 @@ export const scanCanonicalArtifactInventory = async (): Promise<ArtifactInventorySnapshot> => {const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY;const metadata = await lstat(path);- const { format: rootFormat, digest: rootDigest } = await classifyAndHashRoot(+ const {+ format: rootFormat,+ digest: rootDigest,+ zipSource,+ } = await classifyAndHashRootForInventory(path,metadata.isDirectory(),metadata,options.signal,);- const reader = await readerFactory(- path,- rootFormat,- options.environment ?? {},- options.signal,- );+ let reader: ArtifactReader | undefined;+ let readerCreationFailure: { readonly cause: unknown } | undefined;+ try {+ reader = await readerFactory(+ path,+ rootFormat,+ options.environment ?? {},+ options.signal,+ zipSource,+ );+ } catch (cause: unknown) {+ readerCreationFailure = { cause };+ }const ownedReaders: ArtifactReader[] = reader === undefined ? [] : [reader];let outcome:| {@@ -71,6 +86,7 @@ export const scanCanonicalArtifactInventory = async (}| { readonly kind: "failed"; readonly cause: unknown };try {+ if (readerCreationFailure !== undefined) throw readerCreationFailure.cause;if (reader instanceof AsarArtifactReader)await reader.prepareContainer(rootDigest?.sha256, options.signal);const {@@ -86,6 +102,7 @@ export const scanCanonicalArtifactInventory = async (metadata,rootFormat,rootDigest,+ zipSource,reader,signal: options.signal,nodes,@@ -116,6 +133,24 @@ export const scanCanonicalArtifactInventory = async ();}}+ if (zipSource !== undefined) {+ try {+ await zipSource.handle.close();+ } catch (cleanupCause: unknown) {+ const cleanup = ArtifactReaderFailure.cleanupObservation(+ cleanupCause,+ `root ZIP descriptor for ${path}`,+ );+ cleanupFailure = ArtifactReaderFailure.withCleanup(+ cleanupFailure ??+ (outcome.kind === "failed" ? outcome.cause : cleanupCause),+ cleanup,+ outcome.kind === "completed"+ ? { kind: "artifact-inventory", inventory: outcome.snapshot }+ : undefined,+ );+ }+ }if (cleanupFailure !== undefined) throw cleanupFailure;if (outcome.kind === "failed") throw outcome.cause;return outcome.snapshot;@@ -126,6 +161,7 @@ interface SnapshotBuildInput {readonly metadata: Stats;readonly rootFormat: ArtifactOccurrence["artifact_format"];readonly rootDigest: HashResult | null;+ readonly zipSource: StableRegularFileDescriptor | undefined;readonly reader: ArtifactReader | undefined;readonly signal: AbortSignal | undefined;readonly nodes: Map<string, ArtifactNode>;@@ -137,7 +173,8 @@ interface SnapshotBuildInput {const buildInventorySnapshot = async (input: SnapshotBuildInput,): Promise<ArtifactInventorySnapshot> => {- const { path, metadata, rootFormat, rootDigest, reader, signal } = input;+ const { path, metadata, rootFormat, rootDigest, zipSource, reader, signal } =+ input;materializeDirectoryNodes(input.occurrences, input.nodes);const rootNode = createRootNode({digest: rootDigest,@@ -181,7 +218,7 @@ const buildInventorySnapshot = async (ordinal,}));- await verifyRootDigest(path, rootDigest, signal);+ await verifyRootDigest(path, rootDigest, zipSource, signal);const graphSha256 = artifactGraphDigest({nodes: orderedNodes,@@ -256,10 +293,19 @@ const buildIntegrityContradictions = (const verifyRootDigest = async (path: string,rootDigest: HashResult | null,+ zipSource: StableRegularFileDescriptor | undefined,signal: AbortSignal | undefined,): Promise<void> => {if (rootDigest === null) return;- const verified = await hashStableRootArtifact(path, signal);+ const verified =+ zipSource === undefined+ ? await hashStableRootArtifact(path, signal)+ : await hashStableRootArtifactHandle(+ path,+ zipSource.handle,+ zipSource.initial,+ signal,+ );if (verified.sha256 !== rootDigest.sha256 ||verified.bytes !== rootDigest.bytesdiff --git a/src/browser/ElectronFileScope.ts b/src/browser/ElectronFileScope.tsindex 507edb233..74062fd8d 100644--- a/src/browser/ElectronFileScope.ts+++ b/src/browser/ElectronFileScope.ts@@ -3,6 +3,11 @@ import { open, realpath, stat } from "node:fs/promises";import { isAbsolute, join, sep } from "node:path";import { fileURLToPath } from "node:url";+import {+ admitAsarHeader,+ MAX_ASAR_HEADER_BYTES,+} from "../artifacts/AsarHeader.js";+/** Resolve one local file URL while rejecting remote hosts and encoded separators. */export const authorizedElectronFile = async (value: string,@@ -77,7 +82,6 @@ const authorizedAsarMember = async (type ArchiveIdentity = Pick<Stats, "dev" | "ino" | "size" | "mtimeMs">;// This is a retained-memory budget for JSON headers, shared across all archives.-const MAX_HEADER_BYTES = 16 * 1024 * 1024;const MAX_CACHED_HEADERS = 8;const headerCache = new Map<string,@@ -105,23 +109,8 @@ const asarFiles = async (const opened = await handle.stat();if (!opened.isFile() || archiveIdentity(opened) !== identity)throw new Error("ASAR identity changed before header read");- const prefix = Buffer.alloc(16);- const read = await handle.read(prefix, 0, prefix.length, 0);- if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4)- throw new Error("Truncated or invalid ASAR size pickle");- const headerBytes = prefix.readUInt32LE(4);- const payloadBytes = prefix.readUInt32LE(8);- jsonBytes = prefix.readUInt32LE(12);- if (- headerBytes < 8 ||- headerBytes > opened.size - 8 ||- payloadBytes !== headerBytes - 4 ||- jsonBytes > payloadBytes - 4 ||- jsonBytes > MAX_HEADER_BYTES- )- throw new Error(- "ASAR JSON header leaves the file or exceeds its memory budget",- );+ const admitted = await admitAsarHeader(handle);+ jsonBytes = admitted.jsonBytes;const json = Buffer.alloc(jsonBytes);let offset = 0;while (offset < json.length) {@@ -151,7 +140,7 @@ const asarFiles = async ([...headerCache.values()].reduce((sum, entry) => sum + entry.bytes, 0);while (headerCache.size > MAX_CACHED_HEADERS ||- retainedBytes() > MAX_HEADER_BYTES+ retainedBytes() > MAX_ASAR_HEADER_BYTES)headerCache.delete(headerCache.keys().next().value ?? archive);return files;diff --git a/src/browser/assets/PublishWebScripts.ts b/src/browser/assets/PublishWebScripts.tsindex f3af5b8b6..52439494e 100644--- a/src/browser/assets/PublishWebScripts.ts+++ b/src/browser/assets/PublishWebScripts.ts@@ -76,7 +76,12 @@ export const publishWebScripts = async (bytes: bytes.length,},});- await tree.write("manifest.json", Readable.from([bytes]), digest, signal);+ await tree.write(+ "manifest.json",+ Readable.from([bytes]),+ { sha256: digest, bytes: bytes.byteLength },+ signal,+ );signal?.throwIfAborted();await tree.commit();return result;@@ -109,7 +114,7 @@ const publishPlannedScript = async (tree.write(`files/${relativePath}`,Readable.from([content.bytes]),- content.sha256,+ { sha256: content.sha256, bytes: content.bytes.byteLength },signal,);try {diff --git a/src/filesystem/RegularFile.ts b/src/filesystem/RegularFile.tsindex 13982a896..b9a0914c3 100644--- a/src/filesystem/RegularFile.ts+++ b/src/filesystem/RegularFile.ts@@ -16,6 +16,12 @@ export class NonRegularFileReadError extends Error {}}+/** One admitted file descriptor and the metadata that identified its contents. */+export interface StableRegularFileDescriptor {+ readonly handle: FileHandle;+ readonly initial: Stats;+}+/** Open and verify the selected file descriptor without blocking on pipes. */export const openRegularFile = async (path: string,diff --git a/src/firmware/FirmwarePublication.ts b/src/firmware/FirmwarePublication.tsindex 56b011145..47f9c3a5b 100644--- a/src/firmware/FirmwarePublication.ts+++ b/src/firmware/FirmwarePublication.ts@@ -60,7 +60,7 @@ export const publishFirmwareExtraction = async (context: {await tree.write(entry.relativePath,createReadStream(entry.path),- sha256,+ { sha256, bytes: entry.size },context.signal,);files.push({diff --git a/src/javascript/recovery/WakaruPublication.ts b/src/javascript/recovery/WakaruPublication.tsindex 5b89ebe68..d909f219f 100644--- a/src/javascript/recovery/WakaruPublication.ts+++ b/src/javascript/recovery/WakaruPublication.ts@@ -50,7 +50,12 @@ export const publishWakaruArtifacts = async (const { tree, source, parsed, staging, signal } = context;const write = async (path: string, bytes: Buffer) => {const sha256 = recoveryDigest(bytes);- const file = await tree.write(path, Readable.from([bytes]), sha256, signal);+ const file = await tree.write(+ path,+ Readable.from([bytes]),+ { sha256, bytes: bytes.byteLength },+ signal,+ );return {path: join(tree.outputRoot, file.relativePath),sha256: file.sha256,diff --git a/tests/boundary/filesystem/safeOutputTree.test.ts b/tests/boundary/filesystem/safeOutputTree.test.tsindex 32e3c3d32..87c084640 100644--- a/tests/boundary/filesystem/safeOutputTree.test.ts+++ b/tests/boundary/filesystem/safeOutputTree.test.ts@@ -43,11 +43,10 @@ describe("safe artifact output tree", () => {const output = join(parent, "published");const tree = await SafeOutputTree.create(output);await expect(- tree.write(- "nested/file.txt",- Readable.from(Buffer.from("unexpected")),- "0".repeat(64),- ),+ tree.write("nested/file.txt", Readable.from(Buffer.from("unexpected")), {+ sha256: "0".repeat(64),+ bytes: Buffer.byteLength("unexpected"),+ }),).rejects.toThrow(/disagrees/u);expect(await tree.rollback()).toMatchObject({status: "complete",@@ -64,11 +63,10 @@ describe("safe artifact output tree", () => {const output = join(parent, "published");const tree = await SafeOutputTree.create(output);const bytes = Buffer.from("retained until rollback can retry");- await tree.write(- "nested/file.txt",- Readable.from(bytes),- createHash("sha256").update(bytes).digest("hex"),- );+ await tree.write("nested/file.txt", Readable.from(bytes), {+ sha256: createHash("sha256").update(bytes).digest("hex"),+ bytes: bytes.byteLength,+ });try {await chmod(output, 0o500);const failure = await tree.rollback().catch((cause: unknown) => cause);@@ -90,7 +88,10 @@ describe("safe artifact output tree", () => {const bytes = Buffer.from("visible before seal");const digest = createHash("sha256").update(bytes).digest("hex");- await tree.write("file.txt", Readable.from(bytes), digest);+ await tree.write("file.txt", Readable.from(bytes), {+ sha256: digest,+ bytes: bytes.byteLength,+ });expect(await readFile(join(output, "file.txt"), "utf8")).toBe("visible before seal",);@@ -123,7 +124,10 @@ describe("safe artifact output tree", () => {const bytes = Buffer.from("windows output");const digest = createHash("sha256").update(bytes).digest("hex");- await tree.write("nested/file.txt", Readable.from(bytes), digest);+ await tree.write("nested/file.txt", Readable.from(bytes), {+ sha256: digest,+ bytes: bytes.byteLength,+ });await tree.commit();expect(await readFile(join(output, "nested", "file.txt"), "utf8")).toBe("windows output",