Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
19.9 KB
Raw Blame
/**
 * Pass 2 of a capture (sections 5.5, 5.9, 5.10, 13.1): a streaming POSIX pax
 * tar writer, and the pipeline file -> tar -> zstd -> ORSEAL01 -> temp file.
 * Memory is bounded by a few output blocks and stream buffers whatever the
 * project size.
 */
import { createHash } from "node:crypto";
import { lstat, open, readlink } from "node:fs/promises";
import { join } from "node:path";
import { once } from "node:events";
import { PassThrough, Writable, type Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import { constants as zlibConstants, createZstdCompress } from "node:zlib";

import {
  OPEN_ENTRY_FLAGS,
  compareArchivePaths,
  isSameFileIdentity,
  statKeyPrefix,
  type ManifestSource,
  type ScannedEntry,
} from "./manifest.js";
import { hintGarbageCollection } from "./files.js";
import { SealStream, type SealOptions } from "./seal.js";
import { isOutsideArchivePath, outsideFolderIsReal, outsideSourcePath } from "./outside.js";

export const MANIFEST_MEMBER = "__omnirush__/manifest.json";
export const UNSTABLE_MEMBER = "__omnirush__/unstable.json";
export const UNSTABLE_SCHEMA = "omnirush.archive.unstable.v1";
const TAR_BLOCK = 512;
/** Tar output is assembled in blocks of this size; file content is read straight into them. */
const OUTPUT_BLOCK = 1 << 20;
const MAX_OCTAL_11 = 8 ** 11 - 1;
const PAX_HEADER_NAME = Buffer.from("././@PaxHeader", "ascii");
const ZSTD_LEVEL = 3;
/** Output blocks (64 MiB), or members, between two garbage collection hints. */
const GC_HINT_BLOCKS = 64;
const GC_HINT_MEMBERS = 10_000;

type Typeflag = "0" | "2" | "5" | "x";

function isAscii(bytes: Buffer): boolean {
  for (const byte of bytes) if (byte >= 0x80) return false;
  return true;
}

/** The ustar fallback when a pax record carries the real value: non-ASCII bytes become `_`, cut to 100 bytes. */
function asciiFallback(bytes: Buffer): Buffer {
  const out = Buffer.from(bytes.subarray(0, 100));
  for (let index = 0; index < out.length; index += 1) if (out[index]! >= 0x80) out[index] = 0x5f;
  return out;
}

function writeOctal(block: Buffer, offset: number, width: number, value: number): void {
  const digits = value.toString(8).padStart(width - 1, "0");
  if (digits.length > width - 1) throw new RangeError("tar numeric field overflow");
  block.write(digits, offset, width - 1, "ascii");
  block[offset + width - 1] = 0;
}

function clampMtime(seconds: number): number {
  return Math.min(Math.max(Math.floor(seconds), 0), MAX_OCTAL_11);
}

function ustarHeader(name: Buffer, mode: number, size: number, mtime: number, typeflag: Typeflag, linkname: Buffer | null): Buffer {
  const block = Buffer.alloc(TAR_BLOCK);
  name.copy(block, 0, 0, 100);
  writeOctal(block, 100, 8, mode & 0o7777);
  writeOctal(block, 108, 8, 0);
  writeOctal(block, 116, 8, 0);
  writeOctal(block, 124, 12, size);
  writeOctal(block, 136, 12, mtime);
  block.fill(0x20, 148, 156);
  block.write(typeflag, 156, 1, "ascii");
  linkname?.copy(block, 157, 0, 100);
  block.write("ustar\0", 257, 6, "ascii");
  block.write("00", 263, 2, "ascii");
  writeOctal(block, 329, 8, 0);
  writeOctal(block, 337, 8, 0);
  let sum = 0;
  for (const byte of block) sum += byte;
  block.write(sum.toString(8).padStart(6, "0"), 148, 6, "ascii");
  block[154] = 0;
  block[155] = 0x20;
  return block;
}

/** One `"<len> <key>=<value>\n"` record; `len` counts the whole record, its own digits included. */
function paxRecord(key: string, value: string): string {
  const rest = Buffer.byteLength(` ${key}=${value}\n`, "utf8");
  let digits = 1;
  while (String(rest + digits).length !== digits) digits += 1;
  return `${rest + digits} ${key}=${value}\n`;
}

function paddingFor(size: number): number {
  return (TAR_BLOCK - (size % TAR_BLOCK)) % TAR_BLOCK;
}

export type TarMember = { name: string; mode: number; size: number; mtime: number; typeflag: "0" | "2" | "5"; linkname?: string };

/** The header blocks of one member: a pax `x` header first when the path, target or size needs one. */
export function tarMemberHeader(member: TarMember): Buffer {
  const mtime = clampMtime(member.mtime);
  const records: string[] = [];
  let name: Buffer = Buffer.from(member.name, "utf8");
  if (name.length > 100 || !isAscii(name)) {
    records.push(paxRecord("path", member.name));
    name = asciiFallback(name);
  }
  let linkname: Buffer | null = null;
  if (member.linkname !== undefined) {
    linkname = Buffer.from(member.linkname, "utf8");
    if (linkname.length > 100 || !isAscii(linkname)) {
      records.push(paxRecord("linkpath", member.linkname));
      linkname = asciiFallback(linkname);
    }
  }
  let size = member.size;
  if (size > MAX_OCTAL_11) {
    records.push(paxRecord("size", String(size)));
    size = 0;
  }
  const header = ustarHeader(name, member.mode, size, mtime, member.typeflag, linkname);
  if (records.length === 0) return header;
  const body = Buffer.from(records.join(""), "utf8");
  return Buffer.concat([
    ustarHeader(PAX_HEADER_NAME, 0o644, body.length, mtime, "x", null),
    body,
    Buffer.alloc(paddingFor(body.length)),
    header,
  ]);
}

export type PackInput = {
  root: string;
  /** `__omnirush__/manifest.json`: its bytes, or a source that serialises it while it is written. */
  manifest: Buffer | ManifestSource;
  /** Capture v2: members written right after the manifest (`__omnirush__/state.json`). */
  extraMembers?: ReadonlyArray<{ name: string; content: Buffer }>;
  /** created_at in seconds: the mtime of the manifest and unstable members. */
  createdAtSeconds: number;
  /** Exactly the manifest's `files`, in the same order. */
  entries: readonly ScannedEntry[];
  /** Stops the writer between members and output blocks: it then rejects with the signal's reason. */
  signal?: AbortSignal;
  /** Called with tar bytes emitted so far; used by the archive queue's disk admission checks. */
  onOutputBytes?: (bytes: number) => Promise<void>;
};

export type PackOutcome = { unstable: string[]; tarBytes: number };

export type TarPack = { stream: Readable; outcome: () => PackOutcome };

/** Takes a filled output block and resolves with the next (empty) one to fill. */
type EmitBlock = (block: Buffer) => Promise<Buffer>;

/**
 * Writes the tar stream of an archive into output blocks: the manifest, the
 * entries streamed from disk, then `__omnirush__/unstable.json` when an entry
 * changed after pass 1. File content is read straight into the blocks. Only
 * a full block waits on the consumer; the members themselves are plain async
 * calls (async generators, one chain per member, cost several times the
 * memory on a 200k-member tree).
 */
class TarWriter {
  private readonly unstable: string[] = [];
  private tarBytes = 0;
  private finished = false;
  private block: Buffer;
  private used = 0;
  private rotations = 0;
  /** The directories above the last entry read, from the root down, and whether each was a real directory. */
  private readonly parents: Array<{ name: string; real: boolean }> = [];

  constructor(private readonly input: PackInput, first: Buffer, private readonly emit: EmitBlock) {
    this.block = first;
  }

  outcome(): PackOutcome {
    if (!this.finished) throw new Error("the tar stream has not finished");
    return { unstable: [...this.unstable], tarBytes: this.tarBytes };
  }

  private async rotate(): Promise<void> {
    this.input.signal?.throwIfAborted();
    this.tarBytes += this.used;
    await this.input.onOutputBytes?.(this.tarBytes);
    this.block = await this.emit(this.block);
    this.used = 0;
    this.rotations += 1;
    if (this.rotations % GC_HINT_BLOCKS === 0) hintGarbageCollection();
  }

  private async put(bytes: Buffer): Promise<void> {
    let offset = 0;
    while (offset < bytes.length) {
      if (this.used === OUTPUT_BLOCK) await this.rotate();
      const take = Math.min(bytes.length - offset, OUTPUT_BLOCK - this.used);
      bytes.copy(this.block, this.used, offset, offset + take);
      this.used += take;
      offset += take;
    }
  }

  private async zeros(count: number): Promise<void> {
    let left = count;
    while (left > 0) {
      if (this.used === OUTPUT_BLOCK) await this.rotate();
      const take = Math.min(left, OUTPUT_BLOCK - this.used);
      this.block.fill(0, this.used, this.used + take);
      this.used += take;
      left -= take;
    }
  }

  private async smallFile(name: string, content: Buffer | ManifestSource): Promise<void> {
    const source = Buffer.isBuffer(content) ? { size: content.length, chunks: () => [content] } : content;
    await this.put(tarMemberHeader({ name, mode: 0o644, size: source.size, mtime: this.input.createdAtSeconds, typeflag: "0" }));
    let written = 0;
    for (const piece of source.chunks()) {
      written += piece.length;
      if (written > source.size) break;
      await this.put(piece);
    }
    if (written !== source.size) throw new Error(`${name} changed while it was written`);
    await this.zeros(paddingFor(source.size));
  }

  /**
   * Whether every directory above `parts` (a root-relative path, split) is
   * still a real directory and not a symlink, so that opening the path cannot
   * leave the root. Each directory is lstat'ed once, parents first, when the
   * walk enters it, and the result holds for its subtree (contiguous in byte
   * order). A swap after that is caught per file by fileContent()'s identity
   * check.
   */
  private async parentsInRoot(parts: readonly string[]): Promise<boolean> {
    const { parents } = this;
    const depth = parts.length - 1;
    let kept = 0;
    while (kept < parents.length && kept < depth && parents[kept]!.name === parts[kept]) kept += 1;
    parents.length = kept;
    for (let index = kept; index < depth; index += 1) {
      let real = index === 0 || parents[index - 1]!.real;
      if (real) {
        try {
          real = (await lstat(join(this.input.root, ...parts.slice(0, index + 1)))).isDirectory();
        } catch {
          real = false;
        }
      }
      parents.push({ name: parts[index]!, real });
    }
    return depth === 0 || parents[depth - 1]!.real;
  }

  /**
   * Streams exactly entry.size bytes (truncated or NUL-padded); false when the
   * file is not what pass 1 saw. Bytes are copied only from the regular file
   * pass 1 lstat'ed (same st_dev and st_ino) under real directories; anything
   * else (a file replaced, reached through a directory swapped for a symlink,
   * a FIFO) is zero-filled.
   */
  private async fileContent(entry: ScannedEntry, absolute: string, parentsInRoot: boolean): Promise<boolean> {
    let stable = false;
    let handle: Awaited<ReturnType<typeof open>> | null = null;
    if (parentsInRoot) {
      try {
        handle = await open(absolute, OPEN_ENTRY_FLAGS);
        const stats = await handle.stat({ bigint: true });
        if (stats.isFile() && isSameFileIdentity(entry, stats)) {
          stable = stats.size === BigInt(entry.size) && entry.statKey.startsWith(statKeyPrefix(stats.size, stats.mtimeNs));
        } else {
          await handle.close();
          handle = null;
        }
      } catch {
        await handle?.close().catch(() => undefined);
        handle = null;
      }
    }
    const hash = createHash("sha256");
    let remaining = entry.size;
    try {
      while (handle && remaining > 0) {
        if (this.used === OUTPUT_BLOCK) await this.rotate();
        const want = Math.min(remaining, OUTPUT_BLOCK - this.used);
        let bytesRead = 0;
        try {
          ({ bytesRead } = await handle.read(this.block, this.used, want, null));
        } catch {
          stable = false;
        }
        if (bytesRead === 0) break;
        hash.update(this.block.subarray(this.used, this.used + bytesRead));
        this.used += bytesRead;
        remaining -= bytesRead;
      }
    } finally {
      await handle?.close().catch(() => undefined);
    }
    if (remaining > 0) {
      stable = false;
      await this.zeros(remaining);
    } else if (hash.digest("hex") !== entry.sha256) {
      stable = false;
    }
    return stable;
  }

  /** Writes the whole archive; the last, partial block is emitted too. */
  async write(): Promise<void> {
    const { input, unstable } = this;
    await this.smallFile(MANIFEST_MEMBER, input.manifest);
    for (const member of input.extraMembers ?? []) await this.smallFile(member.name, member.content);
    let members = 0;
    for (const entry of input.entries) {
      input.signal?.throwIfAborted();
      members += 1;
      if (members % GC_HINT_MEMBERS === 0) hintGarbageCollection();
      if (entry.type === "dir") {
        await this.put(tarMemberHeader({ name: `${entry.path}/`, mode: entry.mode, size: 0, mtime: entry.mtime, typeflag: "5" }));
        continue;
      }
      if (entry.type === "file" && entry.content) {
        // Bytes prepared in pass 1 (a scrubbed git config).
        await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: entry.content.length, mtime: entry.mtime, typeflag: "0" }));
        await this.put(entry.content);
        await this.zeros(paddingFor(entry.content.length));
        continue;
      }
      const parts = entry.path.split("/");
      if (entry.type === "file" && entry.source) {
        // A staged file (an attachment) read from its own absolute path under the state dir.
        await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: entry.size, mtime: entry.mtime, typeflag: "0" }));
        const stable = await this.fileContent(entry, entry.source, true);
        await this.zeros(paddingFor(entry.size));
        if (!stable) unstable.push(entry.path);
        continue;
      }
      // A file outside the workspace (`__outside__/...`) is read from its own absolute path.
      const outside = isOutsideArchivePath(entry.path);
      const source = outside ? outsideSourcePath(entry.path) : null;
      const absolute = source ?? join(input.root, ...parts);
      const parentsInRoot = outside ? source !== null && entry.type === "file" && (await outsideFolderIsReal(source)) : await this.parentsInRoot(parts);
      if (entry.type === "symlink") {
        const target = entry.target ?? "";
        await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: 0, mtime: entry.mtime, typeflag: "2", linkname: target }));
        try {
          const stats = parentsInRoot ? await lstat(absolute) : null;
          if (!stats?.isSymbolicLink() || (await readlink(absolute)) !== target) unstable.push(entry.path);
        } catch {
          unstable.push(entry.path);
        }
        continue;
      }
      await this.put(tarMemberHeader({ name: entry.path, mode: entry.mode, size: entry.size, mtime: entry.mtime, typeflag: "0" }));
      const stable = await this.fileContent(entry, absolute, parentsInRoot);
      await this.zeros(paddingFor(entry.size));
      if (!stable) unstable.push(entry.path);
    }
    if (unstable.length > 0) {
      unstable.sort(compareArchivePaths);
      await this.smallFile(UNSTABLE_MEMBER, Buffer.from(JSON.stringify({ schema: UNSTABLE_SCHEMA, paths: unstable }), "utf8"));
    }
    await this.zeros(2 * TAR_BLOCK);
    if (this.used > 0) {
      this.tarBytes += this.used;
      await this.input.onOutputBytes?.(this.tarBytes);
      await this.emit(this.block.subarray(0, this.used));
      this.used = 0;
    }
    this.finished = true;
  }
}

/** The tar stream of an archive as a Readable (fresh blocks: safe for any consumer). */
export function packArchiveTar(input: PackInput): TarPack {
  const stream = new PassThrough({ highWaterMark: OUTPUT_BLOCK });
  const writer = new TarWriter(input, Buffer.allocUnsafe(OUTPUT_BLOCK), async (block) => {
    if (!stream.write(block)) await once(stream, "drain");
    return Buffer.allocUnsafe(OUTPUT_BLOCK);
  });
  writer.write().then(() => stream.end(), (error: unknown) => stream.destroy(error instanceof Error ? error : new Error(String(error))));
  return { stream, outcome: () => writer.outcome() };
}

/**
 * A few output blocks recycled between the tar writer and the compressor: a
 * block returns to the pool once the compressor's write callback says it has
 * consumed it, so pass 2 allocates the same few MiB whatever the archive size.
 */
class BlockPool {
  private readonly free: Buffer[] = [];
  private readonly waiters: Array<(block: Buffer) => void> = [];
  private readonly owned = new Map<ArrayBufferLike, Buffer>();

  constructor(private readonly capacity: number) {}

  acquire(): Promise<Buffer> {
    const block = this.free.pop();
    if (block) return Promise.resolve(block);
    if (this.owned.size < this.capacity) {
      const fresh = Buffer.allocUnsafeSlow(OUTPUT_BLOCK);
      this.owned.set(fresh.buffer, fresh);
      return Promise.resolve(fresh);
    }
    return new Promise((resolvePromise) => this.waiters.push(resolvePromise));
  }

  /** Takes back the block a (possibly partial) view belongs to. */
  release(view: Buffer): void {
    const block = this.owned.get(view.buffer);
    if (!block) return;
    const waiter = this.waiters.shift();
    if (waiter) waiter(block);
    else this.free.push(block);
  }

  /** After a failure: nobody may wait forever for a block. */
  unblock(): void {
    for (const waiter of this.waiters.splice(0)) waiter(Buffer.allocUnsafe(OUTPUT_BLOCK));
  }
}

const POOL_BLOCKS = 4;

type FileHandle = Awaited<ReturnType<typeof open>>;

/** The last stage: writes each chunk to the file and waits for it, then fsyncs. */
class FileSink extends Writable {
  constructor(private readonly handle: FileHandle) {
    super({ highWaterMark: OUTPUT_BLOCK });
  }

  override _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void {
    this.writeAll([chunk]).then(() => callback(), (error: unknown) => callback(error instanceof Error ? error : new Error(String(error))));
  }

  override _writev(chunks: Array<{ chunk: Buffer }>, callback: (error?: Error | null) => void): void {
    this.writeAll(chunks.map((item) => item.chunk)).then(() => callback(), (error: unknown) => callback(error instanceof Error ? error : new Error(String(error))));
  }

  private async writeAll(buffers: Buffer[]): Promise<void> {
    let pending = buffers.filter((buffer) => buffer.length > 0);
    while (pending.length > 0) {
      let { bytesWritten } = await this.handle.writev(pending);
      while (pending.length > 0 && bytesWritten >= pending[0]!.length) {
        bytesWritten -= pending[0]!.length;
        pending = pending.slice(1);
      }
      if (pending.length > 0 && bytesWritten > 0) pending = [pending[0]!.subarray(bytesWritten), ...pending.slice(1)];
    }
  }

  override _final(callback: (error?: Error | null) => void): void {
    this.handle.sync().then(() => callback(), (error: unknown) => callback(error instanceof Error ? error : new Error(String(error))));
  }
}

export type SealedArchive = PackOutcome & { size: number; sha256: string; kid: string };

/**
 * file -> tar -> zstd (level 3, checksum, one frame) -> ORSEAL01 -> `outputPath`,
 * then fsync. The output file must not exist yet.
 */
export async function writeSealedArchive(input: PackInput, seal: SealOptions, outputPath: string): Promise<SealedArchive> {
  const sealer = new SealStream(seal);
  const compressor = createZstdCompress({
    chunkSize: 256 * 1024,
    params: {
      [zlibConstants.ZSTD_c_compressionLevel]: ZSTD_LEVEL,
      [zlibConstants.ZSTD_c_checksumFlag]: 1,
    },
  });
  const handle = await open(outputPath, "wx", 0o600);
  try {
    const pool = new BlockPool(POOL_BLOCKS);
    let failure: unknown = null;
    const downstream = pipeline(compressor, sealer, new FileSink(handle));
    downstream.catch((error: unknown) => {
      failure ??= error;
      pool.unblock();
    });
    const writer = new TarWriter(input, await pool.acquire(), async (block) => {
      if (failure) throw failure;
      compressor.write(block, () => pool.release(block));
      return pool.acquire();
    });
    try {
      await writer.write();
    } catch (error) {
      failure ??= error;
    }
    if (failure) {
      compressor.destroy();
      await downstream.catch(() => undefined);
      throw failure;
    }
    compressor.end();
    await downstream;
    return { ...writer.outcome(), ...sealer.result(), kid: sealer.kid };
  } finally {
    await handle.close();
  }
}