Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(artifacts): admit bounded reads and preserve source identity

morluto6d4cefc4f74e
diff --git a/src/application/AppBundleExecutable.ts b/src/application/AppBundleExecutable.ts
index 7df3ca03a..0ae1c073e 100644
--- a/src/application/AppBundleExecutable.ts
+++ b/src/application/AppBundleExecutable.ts
@@ -1,5 +1,5 @@
import { execFile } from "node:child_process";
-import { lstat, readdir, readFile, realpath, stat } from "node:fs/promises";
+import { lstat, opendir, readFile, realpath, stat } from "node:fs/promises";
import { join } from "node:path";
import { promisify } from "node:util";
@@ -156,13 +156,14 @@ const wrappedBundle = async (bundle: string): Promise<string | undefined> => {
if (isAbsence(cause)) return undefined;
throw cause;
}
- const apps = (await readdir(wrapper, { withFileTypes: true })).filter(
- (entry) => entry.isDirectory() && entry.name.toLowerCase().endsWith(".app"),
- );
- const [app] = apps;
- return apps.length === 1 && app !== undefined
- ? join(wrapper, app.name)
- : undefined;
+ let app: string | undefined;
+ for await (const entry of await opendir(wrapper)) {
+ if (!entry.isDirectory() || !entry.name.toLowerCase().endsWith(".app"))
+ continue;
+ if (app !== undefined) return undefined;
+ app = join(wrapper, entry.name);
+ }
+ return app;
};
const resolveLayoutExecutable = async (
diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts
index 3163d17b3..8b532b0d5 100644
--- a/src/application/BinaryTargetResolver.ts
+++ b/src/application/BinaryTargetResolver.ts
@@ -1,4 +1,4 @@
-import { constants } from "node:fs";
+import { constants, type Stats } from "node:fs";
import { createHash } from "node:crypto";
import {
access,
@@ -248,7 +248,7 @@ const isArchiveFormat = (
const sha256Handle = async (
handle: FileHandle,
path: string,
- initial: Awaited<ReturnType<FileHandle["stat"]>>,
+ initial: Stats,
signal?: AbortSignal,
): Promise<string> => {
const hash = createHash("sha256");
@@ -256,11 +256,18 @@ const sha256Handle = async (
let position = 0;
while (true) {
throwIfTargetResolutionCancelled(signal);
- const observed = await handle.read(chunk, 0, chunk.length, position);
+ const observed = await handle.read(
+ chunk,
+ 0,
+ Math.min(chunk.length, initial.size - position + 1),
+ position,
+ );
throwIfTargetResolutionCancelled(signal);
if (observed.bytesRead === 0) break;
hash.update(chunk.subarray(0, observed.bytesRead));
position += observed.bytesRead;
+ if (position > initial.size)
+ throw new BinaryTargetError(path, "target grew while being read");
}
throwIfTargetResolutionCancelled(signal);
const [opened, currentPath] = await Promise.all([handle.stat(), lstat(path)]);
diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts
index 0ccca760a..916d4fd61 100644
--- a/src/application/JsonFiles.ts
+++ b/src/application/JsonFiles.ts
@@ -42,7 +42,7 @@ export const readJsonFile = async (
if (!decoded.ok) {
return err(
new EvidenceFileError("read", "invalid-json", {
- cause: decoded.cause,
+ cause: decoded.cause ?? new SyntaxError(decoded.error),
path: requestedPath,
}),
);
diff --git a/src/artifacts/AsarArtifactReader.ts b/src/artifacts/AsarArtifactReader.ts
index bedd9a5f2..80e9fec0f 100644
--- a/src/artifacts/AsarArtifactReader.ts
+++ b/src/artifacts/AsarArtifactReader.ts
@@ -27,6 +27,7 @@ import {
type ArtifactReader,
} from "./ArtifactReader.js";
import { closeAsarHandle, readValidatedAsarEntry } from "./AsarEntryStream.js";
+import { admitAsarHeader } from "./AsarHeader.js";
/**
* Official Electron ASAR adapter with range-streamed member reads.
@@ -57,6 +58,17 @@ export class AsarArtifactReader implements ArtifactReader {
const snapshotPath = this.#snapshotPath();
this.#resetArchiveState();
this.#entries.clear();
+ const headerHandle = await open(
+ snapshotPath,
+ constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,
+ );
+ try {
+ const admitted = await admitAsarHeader(headerHandle);
+ this.#archiveSize = admitted.archiveBytes;
+ } finally {
+ await headerHandle.close();
+ }
+ abortIfNeeded(signal);
paths = listPackage(snapshotPath, { isPack: false }).sort((left, right) =>
left.localeCompare(right, "en"),
);
diff --git a/src/artifacts/AsarHeader.ts b/src/artifacts/AsarHeader.ts
new file mode 100644
index 000000000..7c17f1cc7
--- /dev/null
+++ b/src/artifacts/AsarHeader.ts
@@ -0,0 +1,48 @@
+import type { FileHandle } from "node:fs/promises";
+
+import { ArtifactReaderFailure } from "./ArtifactReader.js";
+
+/** Shared resource budget for the eagerly decoded ASAR header, not member bytes. */
+export const MAX_ASAR_HEADER_BYTES = 16 * 1024 * 1024;
+
+/** Admit both Chromium pickle extents before an upstream decoder allocates them. */
+export const admitAsarHeader = async (
+ handle: FileHandle,
+): Promise<{
+ readonly archiveBytes: number;
+ readonly headerBytes: number;
+ readonly jsonBytes: number;
+}> => {
+ const observed = await handle.stat();
+ if (!observed.isFile())
+ throw new ArtifactReaderFailure(
+ "format",
+ "ASAR container is not a regular file",
+ );
+ const prefix = Buffer.alloc(16);
+ const read = await handle.read(prefix, 0, prefix.length, 0);
+ if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4)
+ throw new ArtifactReaderFailure(
+ "format",
+ "Truncated or invalid ASAR size pickle",
+ );
+ const headerBytes = prefix.readUInt32LE(4);
+ const payloadBytes = prefix.readUInt32LE(8);
+ const jsonBytes = prefix.readUInt32LE(12);
+ if (
+ headerBytes < 8 ||
+ headerBytes > observed.size - 8 ||
+ payloadBytes !== headerBytes - 4 ||
+ jsonBytes > payloadBytes - 4
+ )
+ throw new ArtifactReaderFailure(
+ "format",
+ "ASAR header leaves its container or has invalid pickle extents",
+ );
+ if (headerBytes > MAX_ASAR_HEADER_BYTES)
+ throw new ArtifactReaderFailure(
+ "limit",
+ `ASAR header exceeds the ${String(MAX_ASAR_HEADER_BYTES)} byte decode budget`,
+ );
+ return { archiveBytes: observed.size, headerBytes, jsonBytes };
+};
diff --git a/src/artifacts/DirectoryArtifactReader.ts b/src/artifacts/DirectoryArtifactReader.ts
index a9e339be5..123a1cb09 100644
--- a/src/artifacts/DirectoryArtifactReader.ts
+++ b/src/artifacts/DirectoryArtifactReader.ts
@@ -3,15 +3,13 @@ import { lstat, open, opendir, realpath } from "node:fs/promises";
import { isAbsolute, join, relative, sep } from "node:path";
import type { Readable } from "node:stream";
-import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
-
import {
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactReader,
} from "./ArtifactReader.js";
-/** Deterministic, non-symlink-following directory reader. */
+/** Non-symlink-following directory reader. */
export class DirectoryArtifactReader implements ArtifactReader {
readonly format = "directory" as const;
readonly #rootPromise: Promise<string>;
@@ -38,13 +36,52 @@ export class DirectoryArtifactReader implements ArtifactReader {
await withDirectoryIoContext("inspect directory", directory, () =>
assertContainedDirectory(root, directory),
);
- const children: string[] = [];
const handle = await opendir(directory).catch((cause: unknown) => {
throw directoryIoFailure("open directory", directory, cause);
});
+ const directories: string[] = [];
try {
- for await (const child of handle) children.push(child.name);
+ for await (const child of handle) {
+ abortIfNeeded(signal);
+ const absolute = join(directory, child.name);
+ const metadata = await lstat(absolute).catch((cause: unknown) => {
+ throw directoryIoFailure("inspect entry", absolute, cause);
+ });
+ const path = relative(root, absolute).split(sep).join("/");
+ const kind = metadata.isSymbolicLink()
+ ? "symlink"
+ : metadata.isDirectory()
+ ? "directory"
+ : metadata.isFile()
+ ? "file"
+ : undefined;
+ if (kind === undefined) continue;
+ yield {
+ path,
+ kind,
+ declaredSize: kind === "file" ? metadata.size : null,
+ compressedSize: null,
+ executable: (metadata.mode & 0o111) !== 0,
+ encrypted: false,
+ byteOffset: null,
+ declaredSha256: null,
+ unpacked: false,
+ limitations:
+ kind === "symlink"
+ ? ["Symlink target was not followed or disclosed."]
+ : [],
+ adapterKey: absolute,
+ ...(kind === "file"
+ ? {
+ sourceIdentity: { device: metadata.dev, inode: metadata.ino },
+ }
+ : {}),
+ };
+ if (kind === "directory" && (descendInto?.(path) ?? true))
+ directories.push(absolute);
+ }
} catch (cause: unknown) {
+ if (cause instanceof ArtifactReaderFailure) throw cause;
throw directoryIoFailure("read directory", directory, cause);
} finally {
// best-effort cleanup: directory-handle close must not mask traversal.
@@ -53,45 +90,6 @@ export class DirectoryArtifactReader implements ArtifactReader {
await withDirectoryIoContext("inspect directory", directory, () =>
assertContainedDirectory(root, directory),
);
- children.sort(compareUnicodeCodePoints);
- const directories: string[] = [];
- for (const name of children) {
- abortIfNeeded(signal);
- const absolute = join(directory, name);
- const metadata = await lstat(absolute).catch((cause: unknown) => {
- throw directoryIoFailure("inspect entry", absolute, cause);
- });
- const path = relative(root, absolute).split(sep).join("/");
- const kind = metadata.isSymbolicLink()
- ? "symlink"
- : metadata.isDirectory()
- ? "directory"
- : metadata.isFile()
- ? "file"
- : undefined;
- if (kind === undefined) continue;
- if (kind === "directory" && (descendInto?.(path) ?? true))
- directories.push(absolute);
- yield {
- path,
- kind,
- declaredSize: kind === "file" ? metadata.size : null,
- compressedSize: null,
- executable: (metadata.mode & 0o111) !== 0,
- encrypted: false,
- byteOffset: null,
- declaredSha256: null,
- unpacked: false,
- limitations:
- kind === "symlink"
- ? ["Symlink target was not followed or disclosed."]
- : [],
- adapterKey: absolute,
- ...(kind === "file"
- ? { sourceIdentity: { device: metadata.dev, inode: metadata.ino } }
- : {}),
- };
- }
directories.reverse();
pending.push(...directories);
}
diff --git a/src/artifacts/SafeOutputTree.ts b/src/artifacts/SafeOutputTree.ts
index 05ce0ac68..d9610192e 100644
--- a/src/artifacts/SafeOutputTree.ts
+++ b/src/artifacts/SafeOutputTree.ts
@@ -105,7 +105,26 @@ export class SafeOutputTree {
}
return new SafeOutputTree(canonicalOutput, platform);
} catch (cause: unknown) {
- await rm(canonicalOutput, { recursive: true, force: true });
+ let removalFailure: unknown;
+ try {
+ await rm(canonicalOutput, { recursive: true, force: true });
+ } catch (cleanupCause: unknown) {
+ removalFailure = cleanupCause;
+ }
+ let absent = false;
+ try {
+ absent = await isAbsent(canonicalOutput);
+ } catch (cleanupCause: unknown) {
+ removalFailure ??= cleanupCause;
+ }
+ if (!absent)
+ throw ArtifactReaderFailure.withCleanup(cause, {
+ reason:
+ removalFailure instanceof Error
+ ? removalFailure.message
+ : "Extraction output root remains after setup failure",
+ resources: [canonicalOutput],
+ });
throw cause;
}
}
@@ -122,56 +141,84 @@ export class SafeOutputTree {
async write(
relativePath: string,
source: Readable,
- expectedSha256: string,
+ expected: { readonly sha256: string; readonly bytes: number },
signal?: AbortSignal,
): Promise<SafeOutputFile> {
- this.#assertWritable();
- const path = normalizeArtifactPath(relativePath);
- this.#registry.add(path, "file");
- const destination = await this.#prepareParent(path);
- const handle = await open(
- destination,
- constants.O_CREAT |
- constants.O_EXCL |
- constants.O_WRONLY |
- constants.O_NOFOLLOW,
- 0o600,
- ).catch(async (cause: unknown) => {
- await throwIfDestinationCaseCollision(dirname(destination), path, cause);
- throw new ArtifactReaderFailure(
- "path",
- `Could not exclusively create extraction path: ${path}`,
- { cause },
- );
- });
- const hash = createHash("sha256");
- let bytes = 0;
try {
- for await (const raw of source) {
- abortIfNeeded(signal);
- const chunk = streamChunkToBuffer(raw);
- bytes += chunk.length;
- hash.update(chunk);
- await writeAll(handle, chunk);
- }
- const sha256 = hash.digest("hex");
- if (sha256 !== expectedSha256)
+ this.#assertWritable();
+ if (!Number.isSafeInteger(expected.bytes) || expected.bytes < 0)
throw new ArtifactReaderFailure(
- "integrity",
- `Extracted content disagrees with inventory: ${path}`,
+ "format",
+ `Invalid expected extraction size for ${relativePath}`,
+ );
+ const path = normalizeArtifactPath(relativePath);
+ this.#registry.add(path, "file");
+ const destination = await this.#prepareParent(path);
+ const handle = await open(
+ destination,
+ constants.O_CREAT |
+ constants.O_EXCL |
+ constants.O_WRONLY |
+ constants.O_NOFOLLOW,
+ 0o600,
+ ).catch(async (cause: unknown) => {
+ await throwIfDestinationCaseCollision(
+ dirname(destination),
+ path,
+ cause,
);
- await handle.sync();
- await handle.close();
- const readback = await hashFile(destination, bytes, signal);
- if (readback.sha256 !== sha256 || readback.bytes !== bytes)
throw new ArtifactReaderFailure(
- "integrity",
- `Durable readback verification failed: ${path}`,
+ "path",
+ `Could not exclusively create extraction path: ${path}`,
+ { cause },
);
- return { relativePath: path, sha256, bytesWritten: bytes };
+ });
+ const hash = createHash("sha256");
+ let bytes = 0;
+ try {
+ for await (const raw of source) {
+ abortIfNeeded(signal);
+ const chunk = streamChunkToBuffer(raw);
+ if (chunk.length > expected.bytes - bytes)
+ throw new ArtifactReaderFailure(
+ "integrity",
+ `Extracted content exceeds the inventoried size: ${path}`,
+ );
+ bytes += chunk.length;
+ hash.update(chunk);
+ await writeAll(handle, chunk);
+ }
+ if (bytes !== expected.bytes)
+ throw new ArtifactReaderFailure(
+ "integrity",
+ `Extracted content size disagrees with inventory: ${path}`,
+ );
+ const sha256 = hash.digest("hex");
+ if (sha256 !== expected.sha256)
+ throw new ArtifactReaderFailure(
+ "integrity",
+ `Extracted content disagrees with inventory: ${path}`,
+ );
+ await handle.sync();
+ await handle.close();
+ const readback = await hashFile(destination, bytes, signal);
+ if (readback.sha256 !== sha256 || readback.bytes !== bytes)
+ throw new ArtifactReaderFailure(
+ "integrity",
+ `Durable readback verification failed: ${path}`,
+ );
+ return { relativePath: path, sha256, bytesWritten: bytes };
+ } catch (cause: unknown) {
+ // best-effort cleanup: file-handle close must not mask the write failure.
+ await handle.close().catch(() => undefined);
+ throw cause;
+ }
} catch (cause: unknown) {
- // best-effort cleanup: file-handle close must not mask the write failure.
- await handle.close().catch(() => undefined);
+ try {
+ source.destroy();
+ } catch {
+ // Preserve the write refusal or failure as the caller-visible error.
+ }
throw cause;
}
}
diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts
index 3c7a92ea1..17637ca45 100644
--- a/src/artifacts/ZipArtifactReader.ts
+++ b/src/artifacts/ZipArtifactReader.ts
@@ -1,4 +1,4 @@
-import { open, type FileHandle } from "node:fs/promises";
+import type { FileHandle } from "node:fs/promises";
import { PassThrough, type Readable } from "node:stream";
import { once } from "node:events";
@@ -10,20 +10,57 @@ import {
type ArtifactReader,
} from "./ArtifactReader.js";
import type { ZipPackageFormat } from "../domain/zipPackageFormat.js";
+import {
+ NonRegularFileReadError,
+ openRegularFile,
+ sameRegularFileState,
+ type StableRegularFileDescriptor,
+} from "../filesystem/RegularFile.js";
class NodeFileReader extends Reader<string> {
#handle: FileHandle | undefined;
+ #initPromise: Promise<void> | undefined;
+ readonly #ownsHandle: boolean;
constructor(
private readonly path: string,
private readonly maximumReadBytes?: number,
+ private readonly admitted?: StableRegularFileDescriptor,
) {
super(path);
+ this.#ownsHandle = admitted === undefined;
+ }
+
+ override init(): Promise<void> {
+ this.#initPromise ??= this.#initialize();
+ return this.#initPromise;
}
- override async init(): Promise<void> {
- this.#handle = await open(this.path, "r");
- this.size = (await this.#handle.stat()).size;
+ async #initialize(): Promise<void> {
+ try {
+ this.#handle =
+ this.admitted?.handle ??
+ (await openRegularFile(this.path, { symlinks: "reject" }));
+ } catch (cause: unknown) {
+ if (cause instanceof NonRegularFileReadError)
+ throw new ArtifactReaderFailure(
+ "format",
+ `ZIP source is not a regular file: ${this.path}`,
+ { cause },
+ );
+ throw cause;
+ }
+ const metadata = await this.#handle.stat();
+ if (
+ !metadata.isFile() ||
+ (this.admitted !== undefined &&
+ !sameRegularFileState(this.admitted.initial, metadata))
+ )
+ throw new ArtifactReaderFailure(
+ "integrity",
+ `ZIP source identity changed before reading: ${this.path}`,
+ );
+ this.size = metadata.size;
}
override async readUint8Array(
@@ -47,7 +84,10 @@ class NodeFileReader extends Reader<string> {
}
async closeHandle(): Promise<void> {
- await this.#handle?.close();
+ if (!this.#ownsHandle) return;
+ await this.#initPromise?.catch(() => undefined);
+ if (this.#handle === undefined) return;
+ await this.#handle.close();
this.#handle = undefined;
}
}
@@ -64,9 +104,10 @@ export class ZipArtifactReader implements ArtifactReader {
path: string,
format: ZipPackageFormat,
maximumMetadataReadBytes?: number,
+ admitted?: StableRegularFileDescriptor,
) {
this.format = format;
- this.#source = new NodeFileReader(path, maximumMetadataReadBytes);
+ this.#source = new NodeFileReader(path, maximumMetadataReadBytes, admitted);
this.#reader = new ZipReader(this.#source, {
checkSignature: true,
checkOverlappingEntry: true,
diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts
index f47c5c56c..e15a81b44 100644
--- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts
+++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts
@@ -25,6 +25,7 @@ import {
type NibHierarchyNode,
} from "./NibViewHierarchy.js";
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
+import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js";
@@ -49,7 +50,6 @@ export const analyzeInterfaceBuilderBundle = async (input: {
const prototypeKeyOmissions = new Map<string, number>();
let omitted = 0;
let attempted = 0;
- let candidates = 0;
let aggregateInputBytes = 0;
const decodeBudget = new InterfaceBuilderDecodeBudget(
MAX_AGGREGATE_DECODE_BYTES,
@@ -60,15 +60,16 @@ export const analyzeInterfaceBuilderBundle = async (input: {
else unreportedOmittedArchives += 1;
};
try {
- for await (const entry of reader.entries(input.signal)) {
- if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path))
- continue;
- candidates += 1;
- if (candidates > limits.max_documents) {
- omitted += 1;
- recordOmittedArchive(entry.path, "max_documents");
- continue;
- }
+ const selection = await selectInterfaceBuilderCandidates(
+ reader,
+ input.signal,
+ limits.max_documents,
+ );
+ omitted = selection.omitted;
+ for (const entry of selection.omittedSamples)
+ recordOmittedArchive(entry.path, "max_documents");
+ unreportedOmittedArchives += selection.unreportedOmitted;
+ for (const entry of selection.selected) {
try {
const remainingInputBytes =
MAX_AGGREGATE_INPUT_BYTES - aggregateInputBytes;
@@ -167,6 +168,60 @@ export const analyzeInterfaceBuilderBundle = async (input: {
});
};
+/** Select lexically earliest document paths independent of host traversal order. */
+const selectInterfaceBuilderCandidates = async (
+ reader: DirectoryArtifactReader,
+ signal: AbortSignal | undefined,
+ maximumDocuments: number,
+): Promise<{
+ readonly selected: readonly ArtifactEntry[];
+ readonly omittedSamples: readonly ArtifactEntry[];
+ readonly omitted: number;
+ readonly unreportedOmitted: number;
+}> => {
+ const capacity = maximumDocuments + MAX_REPORTED_OMITTED_ARCHIVES;
+ const candidates: ArtifactEntry[] = [];
+ let total = 0;
+ for await (const entry of reader.entries(signal)) {
+ if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path))
+ continue;
+ total += 1;
+ retainCandidate(candidates, entry, capacity);
+ }
+ const omittedSamples = candidates.slice(maximumDocuments, capacity);
+ return {
+ selected: candidates.slice(0, maximumDocuments),
+ omittedSamples,
+ omitted: Math.max(0, total - maximumDocuments),
+ unreportedOmitted: Math.max(
+ 0,
+ total - maximumDocuments - omittedSamples.length,
+ ),
+ };
+};
+
+/** Keep the earliest bounded path set; directory iteration order is host-defined. */
+const retainCandidate = (
+ candidates: ArtifactEntry[],
+ entry: ArtifactEntry,
+ maximum: number,
+): void => {
+ let low = 0;
+ let high = candidates.length;
+ while (low < high) {
+ const middle = Math.floor((low + high) / 2);
+ const existing = candidates[middle];
+ if (
+ existing !== undefined &&
+ compareUnicodeCodePoints(existing.path, entry.path) < 0
+ )
+ low = middle + 1;
+ else high = middle;
+ }
+ candidates.splice(low, 0, entry);
+ if (candidates.length > maximum) candidates.pop();
+};
+
const finalizeHierarchyCoverage = (
result: ReturnType<typeof buildInterfaceBuilderAnalysis>,
counts: {
diff --git a/src/artifacts/apple/KeyedArchiveReader.ts b/src/artifacts/apple/KeyedArchiveReader.ts
index ec50ab4cf..3a24f0962 100644
--- a/src/artifacts/apple/KeyedArchiveReader.ts
+++ b/src/artifacts/apple/KeyedArchiveReader.ts
@@ -147,29 +147,28 @@ export const inspectBundleKeyedArchive = async (input: {
try {
// Match inventory spelling while retaining the raw path for I/O and evidence.
const wanted = selected.path.normalize("NFC");
- const matches: ArtifactEntry[] = [];
- for await (const entry of reader.entries(input.signal, (directory) =>
+ let entry: ArtifactEntry | undefined;
+ for await (const candidate of reader.entries(input.signal, (directory) =>
wanted.startsWith(`${directory.normalize("NFC")}/`),
)) {
- if (entry.path.normalize("NFC") !== wanted) continue;
- if (entry.kind !== "file")
+ if (candidate.path.normalize("NFC") !== wanted) continue;
+ if (candidate.kind !== "file")
throw archivePathError(
"invalid_value",
- `Archive path selects a ${entry.kind}, not a regular file: ${selected.path}`,
+ `Archive path selects a ${candidate.kind}, not a regular file: ${selected.path}`,
);
- if ((entry.declaredSize ?? 0) > MAX_BYTES)
+ if ((candidate.declaredSize ?? 0) > MAX_BYTES)
throw new ArtifactReaderFailure(
"limit",
"Keyed archive exceeds 64 MiB",
);
- matches.push(entry);
+ if (entry !== undefined)
+ throw archivePathError(
+ "invalid_value",
+ `Archive path matches multiple Unicode-equivalent bundle entries; selection is ambiguous: ${selected.path}`,
+ );
+ entry = candidate;
}
- if (matches.length > 1)
- throw archivePathError(
- "invalid_value",
- `Archive path matches ${String(matches.length)} Unicode-equivalent bundle entries; selection is ambiguous: ${selected.path}`,
- );
- const entry = matches[0];
if (entry === undefined)
throw archivePathError(
"invalid_value",
diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts
index 5bf6e94cf..5fcfba720 100644
--- a/src/artifacts/extraction/ArtifactExtraction.ts
+++ b/src/artifacts/extraction/ArtifactExtraction.ts
@@ -215,7 +215,10 @@ const materializeSelection = async ({
const written = await output.write(
path,
stream,
- selectedItem.node.sha256,
+ {
+ sha256: selectedItem.node.sha256,
+ bytes: selectedItem.node.size,
+ },
signal,
);
extracted.push({
diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts
index fe29c6057..6e69c64c8 100644
--- a/src/artifacts/inventory/classify.ts
+++ b/src/artifacts/inventory/classify.ts
@@ -5,6 +5,7 @@ import {
} from "../../filesystem/RegularFile.js";
import type { FileHandle } from "node:fs/promises";
import type { Stats } from "node:fs";
+import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js";
@@ -17,18 +18,46 @@ import { ArtifactReaderFailure } from "../ArtifactReader.js";
import type { HashResult } from "../ArtifactHash.js";
import { hashStableRootArtifactHandle } from "./hashStableRootArtifact.js";
+interface RootClassification {
+ readonly format: ArtifactOccurrence["artifact_format"];
+ readonly digest: HashResult | null;
+}
+
+type RootInventoryClassification = RootClassification & {
+ readonly zipSource: StableRegularFileDescriptor | undefined;
+};
+
/** Classify and hash one file root through the same stable open descriptor. */
export const classifyAndHashRoot = async (
path: string,
directory: boolean,
expectedMetadata: Stats,
signal?: AbortSignal,
-): Promise<{
- readonly format: ArtifactOccurrence["artifact_format"];
- readonly digest: HashResult | null;
-}> => {
- if (directory) return { format: "directory", digest: null };
+): Promise<RootClassification> => {
+ const result = await classifyAndHashRootForInventory(
+ path,
+ directory,
+ expectedMetadata,
+ signal,
+ );
+ try {
+ return { format: result.format, digest: result.digest };
+ } finally {
+ await result.zipSource?.handle.close();
+ }
+};
+
+/** Classify a root and retain its admitted ZIP descriptor for child inventory. */
+export const classifyAndHashRootForInventory = async (
+ path: string,
+ directory: boolean,
+ expectedMetadata: Stats,
+ signal?: AbortSignal,
+): Promise<RootInventoryClassification> => {
+ if (directory)
+ return { format: "directory", digest: null, zipSource: undefined };
const handle = await openRootFile(path, signal);
+ let transferHandle = false;
try {
const initial = await handle.stat();
if (!sameRegularFileState(expectedMetadata, initial))
@@ -43,12 +72,25 @@ export const classifyAndHashRoot = async (
initial,
signal,
);
- return { format, digest };
+ const zipSource = isZipFormat(format);
+ if (zipSource) transferHandle = true;
+ return {
+ format,
+ digest,
+ zipSource: zipSource ? { handle, initial } : undefined,
+ };
} finally {
- await handle.close();
+ if (!transferHandle) await handle.close();
}
};
+const isZipFormat = (format: ArtifactOccurrence["artifact_format"]): boolean =>
+ format === "zip" ||
+ format === "ipa" ||
+ format === "apk" ||
+ format === "msix" ||
+ format === "appx";
+
const openRootFile = async (
path: string,
signal?: AbortSignal,
diff --git a/src/artifacts/inventory/hashStableRootArtifact.ts b/src/artifacts/inventory/hashStableRootArtifact.ts
index e441062d3..cd3cc302c 100644
--- a/src/artifacts/inventory/hashStableRootArtifact.ts
+++ b/src/artifacts/inventory/hashStableRootArtifact.ts
@@ -1,4 +1,5 @@
import { lstat } from "node:fs/promises";
+import type { Stats } from "node:fs";
import type { FileHandle } from "node:fs/promises";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
@@ -45,11 +46,13 @@ export const hashStableRootArtifact = async (
export const hashStableRootArtifactHandle = async (
path: string,
handle: FileHandle,
- initial: Awaited<ReturnType<FileHandle["stat"]>>,
+ initial: Stats,
signal?: AbortSignal,
): Promise<HashResult> => {
const digest = await hashReadable(
- handle.createReadStream({ autoClose: false }),
+ // Read at most the admitted extent plus one byte to detect growth without
+ // following an actively appended file indefinitely.
+ handle.createReadStream({ autoClose: false, start: 0, end: initial.size }),
signal,
);
abortIfNeeded(signal);
diff --git a/src/artifacts/inventory/reader.ts b/src/artifacts/inventory/reader.ts
index 692431a00..21ad5adea 100644
--- a/src/artifacts/inventory/reader.ts
+++ b/src/artifacts/inventory/reader.ts
@@ -6,12 +6,14 @@ import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
import { ZipArtifactReader } from "../ZipArtifactReader.js";
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "../../domain/zipPackageFormat.js";
+import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
export const createReader = async (
path: string,
format: ArtifactOccurrence["artifact_format"],
environment: Readonly<NodeJS.ProcessEnv>,
signal?: AbortSignal,
+ zipSource?: StableRegularFileDescriptor,
): Promise<ArtifactReader | undefined> => {
switch (format) {
case "directory":
@@ -21,7 +23,7 @@ export const createReader = async (
case "apk":
case "msix":
case "appx":
- return new ZipArtifactReader(path, format);
+ return new ZipArtifactReader(path, format, undefined, zipSource);
case "asar":
return new AsarArtifactReader(path);
case "mach-o-universal":
diff --git a/src/artifacts/inventory/scanCanonical.ts b/src/artifacts/inventory/scanCanonical.ts
index 0719e58f4..9ae0db083 100644
--- a/src/artifacts/inventory/scanCanonical.ts
+++ b/src/artifacts/inventory/scanCanonical.ts
@@ -5,6 +5,7 @@ import { AsarArtifactReader } from "../AsarArtifactReader.js";
import { lstat } from "node:fs/promises";
import type { Stats } from "node:fs";
+import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
import {
ArtifactReaderFailure,
@@ -31,9 +32,12 @@ import {
artifactGraphDigest,
artifactManifestId,
} from "../../domain/artifactIdentity.js";
-import { classifyAndHashRoot } from "./classify.js";
+import { classifyAndHashRootForInventory } from "./classify.js";
import { type HashResult } from "../ArtifactHash.js";
-import { hashStableRootArtifact } from "./hashStableRootArtifact.js";
+import {
+ hashStableRootArtifact,
+ hashStableRootArtifactHandle,
+} from "./hashStableRootArtifact.js";
import { createReader, inventoryLimitations } from "./reader.js";
import {
scanReader,
@@ -51,18 +55,29 @@ export const scanCanonicalArtifactInventory = async (
): Promise<ArtifactInventorySnapshot> => {
const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY;
const metadata = await lstat(path);
- const { format: rootFormat, digest: rootDigest } = await classifyAndHashRoot(
+ const {
+ format: rootFormat,
+ digest: rootDigest,
+ zipSource,
+ } = await classifyAndHashRootForInventory(
path,
metadata.isDirectory(),
metadata,
options.signal,
);
- const reader = await readerFactory(
- path,
- rootFormat,
- options.environment ?? {},
- options.signal,
- );
+ let reader: ArtifactReader | undefined;
+ let readerCreationFailure: { readonly cause: unknown } | undefined;
+ try {
+ reader = await readerFactory(
+ path,
+ rootFormat,
+ options.environment ?? {},
+ options.signal,
+ zipSource,
+ );
+ } catch (cause: unknown) {
+ readerCreationFailure = { cause };
+ }
const ownedReaders: ArtifactReader[] = reader === undefined ? [] : [reader];
let outcome:
| {
@@ -71,6 +86,7 @@ export const scanCanonicalArtifactInventory = async (
}
| { readonly kind: "failed"; readonly cause: unknown };
try {
+ if (readerCreationFailure !== undefined) throw readerCreationFailure.cause;
if (reader instanceof AsarArtifactReader)
await reader.prepareContainer(rootDigest?.sha256, options.signal);
const {
@@ -86,6 +102,7 @@ export const scanCanonicalArtifactInventory = async (
metadata,
rootFormat,
rootDigest,
+ zipSource,
reader,
signal: options.signal,
nodes,
@@ -116,6 +133,24 @@ export const scanCanonicalArtifactInventory = async (
);
}
}
+ if (zipSource !== undefined) {
+ try {
+ await zipSource.handle.close();
+ } catch (cleanupCause: unknown) {
+ const cleanup = ArtifactReaderFailure.cleanupObservation(
+ cleanupCause,
+ `root ZIP descriptor for ${path}`,
+ );
+ cleanupFailure = ArtifactReaderFailure.withCleanup(
+ cleanupFailure ??
+ (outcome.kind === "failed" ? outcome.cause : cleanupCause),
+ cleanup,
+ outcome.kind === "completed"
+ ? { kind: "artifact-inventory", inventory: outcome.snapshot }
+ : undefined,
+ );
+ }
+ }
if (cleanupFailure !== undefined) throw cleanupFailure;
if (outcome.kind === "failed") throw outcome.cause;
return outcome.snapshot;
@@ -126,6 +161,7 @@ interface SnapshotBuildInput {
readonly metadata: Stats;
readonly rootFormat: ArtifactOccurrence["artifact_format"];
readonly rootDigest: HashResult | null;
+ readonly zipSource: StableRegularFileDescriptor | undefined;
readonly reader: ArtifactReader | undefined;
readonly signal: AbortSignal | undefined;
readonly nodes: Map<string, ArtifactNode>;
@@ -137,7 +173,8 @@ interface SnapshotBuildInput {
const buildInventorySnapshot = async (
input: SnapshotBuildInput,
): Promise<ArtifactInventorySnapshot> => {
- const { path, metadata, rootFormat, rootDigest, reader, signal } = input;
+ const { path, metadata, rootFormat, rootDigest, zipSource, reader, signal } =
+ input;
materializeDirectoryNodes(input.occurrences, input.nodes);
const rootNode = createRootNode({
digest: rootDigest,
@@ -181,7 +218,7 @@ const buildInventorySnapshot = async (
ordinal,
}));
- await verifyRootDigest(path, rootDigest, signal);
+ await verifyRootDigest(path, rootDigest, zipSource, signal);
const graphSha256 = artifactGraphDigest({
nodes: orderedNodes,
@@ -256,10 +293,19 @@ const buildIntegrityContradictions = (
const verifyRootDigest = async (
path: string,
rootDigest: HashResult | null,
+ zipSource: StableRegularFileDescriptor | undefined,
signal: AbortSignal | undefined,
): Promise<void> => {
if (rootDigest === null) return;
- const verified = await hashStableRootArtifact(path, signal);
+ const verified =
+ zipSource === undefined
+ ? await hashStableRootArtifact(path, signal)
+ : await hashStableRootArtifactHandle(
+ path,
+ zipSource.handle,
+ zipSource.initial,
+ signal,
+ );
if (
verified.sha256 !== rootDigest.sha256 ||
verified.bytes !== rootDigest.bytes
diff --git a/src/browser/ElectronFileScope.ts b/src/browser/ElectronFileScope.ts
index 507edb233..74062fd8d 100644
--- a/src/browser/ElectronFileScope.ts
+++ b/src/browser/ElectronFileScope.ts
@@ -3,6 +3,11 @@ import { open, realpath, stat } from "node:fs/promises";
import { isAbsolute, join, sep } from "node:path";
import { fileURLToPath } from "node:url";
+import {
+ admitAsarHeader,
+ MAX_ASAR_HEADER_BYTES,
+} from "../artifacts/AsarHeader.js";
+
/** Resolve one local file URL while rejecting remote hosts and encoded separators. */
export const authorizedElectronFile = async (
value: string,
@@ -77,7 +82,6 @@ const authorizedAsarMember = async (
type ArchiveIdentity = Pick<Stats, "dev" | "ino" | "size" | "mtimeMs">;
// This is a retained-memory budget for JSON headers, shared across all archives.
-const MAX_HEADER_BYTES = 16 * 1024 * 1024;
const MAX_CACHED_HEADERS = 8;
const headerCache = new Map<
string,
@@ -105,23 +109,8 @@ const asarFiles = async (
const opened = await handle.stat();
if (!opened.isFile() || archiveIdentity(opened) !== identity)
throw new Error("ASAR identity changed before header read");
- const prefix = Buffer.alloc(16);
- const read = await handle.read(prefix, 0, prefix.length, 0);
- if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4)
- throw new Error("Truncated or invalid ASAR size pickle");
- const headerBytes = prefix.readUInt32LE(4);
- const payloadBytes = prefix.readUInt32LE(8);
- jsonBytes = prefix.readUInt32LE(12);
- if (
- headerBytes < 8 ||
- headerBytes > opened.size - 8 ||
- payloadBytes !== headerBytes - 4 ||
- jsonBytes > payloadBytes - 4 ||
- jsonBytes > MAX_HEADER_BYTES
- )
- throw new Error(
- "ASAR JSON header leaves the file or exceeds its memory budget",
- );
+ const admitted = await admitAsarHeader(handle);
+ jsonBytes = admitted.jsonBytes;
const json = Buffer.alloc(jsonBytes);
let offset = 0;
while (offset < json.length) {
@@ -151,7 +140,7 @@ const asarFiles = async (
[...headerCache.values()].reduce((sum, entry) => sum + entry.bytes, 0);
while (
headerCache.size > MAX_CACHED_HEADERS ||
- retainedBytes() > MAX_HEADER_BYTES
+ retainedBytes() > MAX_ASAR_HEADER_BYTES
)
headerCache.delete(headerCache.keys().next().value ?? archive);
return files;
diff --git a/src/browser/assets/PublishWebScripts.ts b/src/browser/assets/PublishWebScripts.ts
index f3af5b8b6..52439494e 100644
--- a/src/browser/assets/PublishWebScripts.ts
+++ b/src/browser/assets/PublishWebScripts.ts
@@ -76,7 +76,12 @@ export const publishWebScripts = async (
bytes: bytes.length,
},
});
- await tree.write("manifest.json", Readable.from([bytes]), digest, signal);
+ await tree.write(
+ "manifest.json",
+ Readable.from([bytes]),
+ { sha256: digest, bytes: bytes.byteLength },
+ signal,
+ );
signal?.throwIfAborted();
await tree.commit();
return result;
@@ -109,7 +114,7 @@ const publishPlannedScript = async (
tree.write(
`files/${relativePath}`,
Readable.from([content.bytes]),
- content.sha256,
+ { sha256: content.sha256, bytes: content.bytes.byteLength },
signal,
);
try {
diff --git a/src/filesystem/RegularFile.ts b/src/filesystem/RegularFile.ts
index 13982a896..b9a0914c3 100644
--- a/src/filesystem/RegularFile.ts
+++ b/src/filesystem/RegularFile.ts
@@ -16,6 +16,12 @@ export class NonRegularFileReadError extends Error {
}
}
+/** One admitted file descriptor and the metadata that identified its contents. */
+export interface StableRegularFileDescriptor {
+ readonly handle: FileHandle;
+ readonly initial: Stats;
+}
+
/** Open and verify the selected file descriptor without blocking on pipes. */
export const openRegularFile = async (
path: string,
diff --git a/src/firmware/FirmwarePublication.ts b/src/firmware/FirmwarePublication.ts
index 56b011145..47f9c3a5b 100644
--- a/src/firmware/FirmwarePublication.ts
+++ b/src/firmware/FirmwarePublication.ts
@@ -60,7 +60,7 @@ export const publishFirmwareExtraction = async (context: {
await tree.write(
entry.relativePath,
createReadStream(entry.path),
- sha256,
+ { sha256, bytes: entry.size },
context.signal,
);
files.push({
diff --git a/src/javascript/recovery/WakaruPublication.ts b/src/javascript/recovery/WakaruPublication.ts
index 5b89ebe68..d909f219f 100644
--- a/src/javascript/recovery/WakaruPublication.ts
+++ b/src/javascript/recovery/WakaruPublication.ts
@@ -50,7 +50,12 @@ export const publishWakaruArtifacts = async (
const { tree, source, parsed, staging, signal } = context;
const write = async (path: string, bytes: Buffer) => {
const sha256 = recoveryDigest(bytes);
- const file = await tree.write(path, Readable.from([bytes]), sha256, signal);
+ const file = await tree.write(
+ path,
+ Readable.from([bytes]),
+ { sha256, bytes: bytes.byteLength },
+ signal,
+ );
return {
path: join(tree.outputRoot, file.relativePath),
sha256: file.sha256,
diff --git a/tests/boundary/filesystem/safeOutputTree.test.ts b/tests/boundary/filesystem/safeOutputTree.test.ts
index 32e3c3d32..87c084640 100644
--- a/tests/boundary/filesystem/safeOutputTree.test.ts
+++ b/tests/boundary/filesystem/safeOutputTree.test.ts
@@ -43,11 +43,10 @@ describe("safe artifact output tree", () => {
const output = join(parent, "published");
const tree = await SafeOutputTree.create(output);
await expect(
- tree.write(
- "nested/file.txt",
- Readable.from(Buffer.from("unexpected")),
- "0".repeat(64),
- ),
+ tree.write("nested/file.txt", Readable.from(Buffer.from("unexpected")), {
+ sha256: "0".repeat(64),
+ bytes: Buffer.byteLength("unexpected"),
+ }),
).rejects.toThrow(/disagrees/u);
expect(await tree.rollback()).toMatchObject({
status: "complete",
@@ -64,11 +63,10 @@ describe("safe artifact output tree", () => {
const output = join(parent, "published");
const tree = await SafeOutputTree.create(output);
const bytes = Buffer.from("retained until rollback can retry");
- await tree.write(
- "nested/file.txt",
- Readable.from(bytes),
- createHash("sha256").update(bytes).digest("hex"),
- );
+ await tree.write("nested/file.txt", Readable.from(bytes), {
+ sha256: createHash("sha256").update(bytes).digest("hex"),
+ bytes: bytes.byteLength,
+ });
try {
await chmod(output, 0o500);
const failure = await tree.rollback().catch((cause: unknown) => cause);
@@ -90,7 +88,10 @@ describe("safe artifact output tree", () => {
const bytes = Buffer.from("visible before seal");
const digest = createHash("sha256").update(bytes).digest("hex");
- await tree.write("file.txt", Readable.from(bytes), digest);
+ await tree.write("file.txt", Readable.from(bytes), {
+ sha256: digest,
+ bytes: bytes.byteLength,
+ });
expect(await readFile(join(output, "file.txt"), "utf8")).toBe(
"visible before seal",
);
@@ -123,7 +124,10 @@ describe("safe artifact output tree", () => {
const bytes = Buffer.from("windows output");
const digest = createHash("sha256").update(bytes).digest("hex");
- await tree.write("nested/file.txt", Readable.from(bytes), digest);
+ await tree.write("nested/file.txt", Readable.from(bytes), {
+ sha256: digest,
+ bytes: bytes.byteLength,
+ });
await tree.commit();
expect(await readFile(join(output, "nested", "file.txt"), "utf8")).toBe(
"windows output",