Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
23.8 KB
Raw Blame
/**
 * BUG-175 — recursive "Download as zip" for a GFS folder. The walk runs in the
 * renderer over the existing `window.clerum.gfs` read plane; the ARCHIVE is
 * assembled streamed in the main process (temp file + native save dialog, see
 * gfs/zipStream.ts), so the renderer never holds archive bytes and its peak
 * memory is one in-flight transfer, independent of folder size (R1-H2, spec:
 * round-2 extensions).
 *
 * Budget contract: the producer meters this walk's request classes per actor —
 * resource reads (children, affordances, resolve) and proxy reads (downloads)
 * — at 480/min each by default (control-api `externalGfsResourceReadRlPerMin`
 * / `externalGfsProxyReadRlPerMin`). Every listing and download below goes
 * through `createGfsReadThrottle`, deliberately spaced at a CONSERVATIVE
 * fraction of those ceilings (see `GFS_ZIP_READS_PER_MINUTE`) so one walk
 * leaves room for the browsing surfaces sharing the same classes, and a 429
 * answer backs off once by the server-provided `retryAfterSeconds` before
 * failing. Byte and entry ceilings bound the walk before anything large is
 * buffered, and entries the caller cannot read (a listing row marked
 * unreadable, or a 403 on the download itself) are skipped and reported back
 * so the result can show a visible skip notice instead of a silently holey
 * archive.
 */
import { isRateLimited, parseHttpStatus, parseRetryAfterSeconds } from '@lib/gfsGrantErrors'
import { ZipEntryNameAllocator, foldEntryName } from '../../../src/gfs/zipEntryName'
import { escapeReservedWindowsDeviceName } from '../../../src/gfs/zipEntryPath'
import { ZIP_MAX_ENTRIES, ZIP_MAX_TOTAL_BYTES } from '../../../src/gfs/zipLimits'

/**
 * Client-side pacing for the walk, a deliberate conservative margin: the
 * producer's default ceilings for the classes used here are 480/min each
 * (resource reads, proxy reads); spacing at 120/min keeps one walk well under
 * them while the Files browsing surface shares the same classes.
 */
export const GFS_ZIP_READS_PER_MINUTE = 120
/**
 * Refuse before buffering: total bytes across the folder. 512 MiB bounds the
 * streamed archive's temp-file size and each transfer's producer-side bound;
 * the RENDERER holds no payload bytes at all (the main process fetches and
 * writes each transfer chunk by chunk), so peak memory is one chunk in main
 * plus the archive's central-directory metadata (spec: round-2 extensions,
 * streamed assembly; R1-H2).
 */
export const GFS_ZIP_MAX_TOTAL_BYTES = ZIP_MAX_TOTAL_BYTES
/** Refuse before buffering: total file entries across the folder. */
export const GFS_ZIP_MAX_ENTRIES = ZIP_MAX_ENTRIES

export interface GfsZipSkippedEntry {
  path: string
  reason: string
}

export type GfsFolderZipPhase = 'listing' | 'downloading' | 'assembling'

export interface GfsFolderZipProgress {
  phase: GfsFolderZipPhase
  filesFound: number
  filesAdded: number
  currentPath: string | null
}

export interface GfsFolderZipResult {
  /** False when the user canceled the native save dialog (temp file deleted). */
  saved: boolean
  savedPath: string | null
  fileName: string
  fileCount: number
  skipped: GfsZipSkippedEntry[]
}

/** The walk exceeded a configured ceiling — a refusal, not a failure. */
export class GfsFolderZipLimitError extends Error {}

/** Nothing was archivable (an empty folder, or every entry was skipped). */
export class GfsFolderZipEmptyError extends Error {
  readonly skipped: GfsZipSkippedEntry[]
  constructor(folderName: string, skipped: GfsZipSkippedEntry[]) {
    super(`"${folderName}" has no downloadable files.`)
    this.name = 'GfsFolderZipEmptyError'
    this.skipped = skipped
  }
}

export interface GfsReadThrottle {
  acquire(): Promise<void>
}

export interface CreateGfsReadThrottleOptions {
  requestsPerMinute?: number
  now?: () => number
  sleep?: (ms: number) => Promise<void>
}

/**
 * Sequential read spacing for the server's per-actor GFS budget: one acquired
 * slot per request, `60_000 / requestsPerMinute` apart. Injectable clock and
 * sleep keep the spacing testable without real time.
 */
export function createGfsReadThrottle(options: CreateGfsReadThrottleOptions = {}): GfsReadThrottle {
  const requestsPerMinute = options.requestsPerMinute ?? GFS_ZIP_READS_PER_MINUTE
  const now = options.now ?? (() => Date.now())
  const sleep = options.sleep ?? (ms => new Promise<void>(resolve => setTimeout(resolve, ms)))
  const intervalMs = 60_000 / requestsPerMinute
  let nextAllowedAt = 0
  return {
    async acquire() {
      const current = now()
      if (current < nextAllowedAt) {
        await sleep(nextAllowedAt - current)
      }
      nextAllowedAt = Math.max(nextAllowedAt, now()) + intervalMs
    },
  }
}

export interface GfsZipChildItem {
  resourceId: string
  gfsUri: string
  name: string
  kind: 'file' | 'directory'
  /** Absent on older servers — treated as 0 by the size guard, never as NaN. */
  bytes?: number
  drive?: string
  readable?: boolean
}

export interface GfsZipChildrenPage {
  items: GfsZipChildItem[]
  nextCursor: string | null
}

export interface GfsZipStreamDeps {
  start(): Promise<{ jobId: string }>
  /**
   * Appends one entry by handing the MAIN process the resource URI: the
   * payload is fetched and written into the archive chunk by chunk, so it
   * never crosses the IPC boundary and never materializes as one buffer
   * (R1-H2). `options.maxBytes` is the producer-side bound for this
   * transfer; `options.signal` stops the producer-side fetch.
   */
  appendFromUri(
    jobId: string,
    name: string,
    uri: string,
    options?: { maxBytes?: number; signal?: AbortSignal }
  ): Promise<{ name: string; bytes: number }>
  finish(
    jobId: string,
    suggestedName: string
  ): Promise<{ saved: boolean; filePath: string | null; entryCount: number }>
  abort(jobId: string): Promise<unknown>
}

export interface GfsFolderZipDeps {
  /**
   * Lists one folder page. `options.signal` propagates the walk's stop to the
   * producer-side fetch when the default bridge is used (R1-M1).
   */
  listChildren(
    resourceId: string,
    drive: string,
    cursor?: string,
    options?: { signal?: AbortSignal }
  ): Promise<GfsZipChildrenPage>
  /**
   * Streamed, file-backed archive assembly in the main process (R1-H2): one
   * append per downloaded file, one finish at the end, abort cleans the temp
   * file on any failure path.
   */
  zipStream: GfsZipStreamDeps
  throttle: GfsReadThrottle
  sleep: (ms: number) => Promise<void>
}

export interface CreateGfsFolderZipOptions {
  onProgress?: (progress: GfsFolderZipProgress) => void
  limits?: { maxTotalBytes?: number; maxEntries?: number }
  deps?: Partial<GfsFolderZipDeps>
  /**
   * Cooperative stop for a walk that can legally run for many minutes: every
   * wait (throttle slot, 429 backoff, in-flight request) races the signal, and
   * each loop turn re-checks it. Aborting rejects with a DOMException named
   * `AbortError` (`isFolderZipAbortError`).
   */
  signal?: AbortSignal
}

export interface GfsZipFolderSource {
  resourceId: string
  drive: string
  name: string
}

/**
 * Complete-entry path bound. ZIP name fields are 16-bit BYTE counts, so the
 * fully assembled entry path (all segments joined) must stay strictly under
 * 65535 encoded bytes. Segments are capped individually
 * (`GFS_ZIP_MAX_SEGMENT_NAME_BYTES`); this bound closes the deep-nesting case
 * where many legal segments stack past the field. An overlong entry is skipped
 * with a visible notice, and an overlong DIRECTORY prunes its subtree — every
 * descendant would overflow too (spec: ZIP budget model).
 */
export const GFS_ZIP_MAX_ENTRY_PATH_BYTES = 65535

const zipPathEncoder = new TextEncoder()

/** Human-noticeable stand-in for an over-long path (the real one is unusable). */
function shortenForNotice(path: string): string {
  return path.length > 80 ? `${path.slice(0, 48)}…${path.slice(-24)}` : path
}

function toMessage(error: unknown): string {
  return error instanceof Error ? error.message : String(error)
}

/** The user stopped the walk — not a failure, and never a save. */
export function isFolderZipAbortError(error: unknown): boolean {
  return error instanceof DOMException && error.name === 'AbortError'
}

function folderZipAbortedError(): DOMException {
  return new DOMException('Folder-zip walk was stopped.', 'AbortError')
}

/** A per-resource denial for THIS entry (403/forbidden), not a session failure. */
function isAccessDenied(message: string): boolean {
  return parseHttpStatus(message) === 403 || message.toLowerCase().includes('forbidden')
}

/**
 * Zip entry-name length ceiling per segment. ZIP name fields are 16-bit byte
 * counts; a name whose UTF-8 encoding reaches 65 536 bytes would silently
 * truncate there and corrupt the archive, so segments are cut well under it.
 */
export const GFS_ZIP_MAX_SEGMENT_NAME_BYTES = 1024

const segmentNameEncoder = new TextEncoder()

/** Truncates by code point so the UTF-8 encoding stays `maxBytes` or less. */
function truncateToUtf8Bytes(name: string, maxBytes: number): string {
  if (segmentNameEncoder.encode(name).length <= maxBytes) return name
  let kept = ''
  let used = 0
  for (const character of name) {
    const size = segmentNameEncoder.encode(character).length
    if (used + size > maxBytes) break
    kept += character
    used += size
  }
  return kept
}

/**
 * Zip-safe path segment. Separators, NUL and the characters Windows forbids
 * (`: * ? " < > |`) map to `_`; trailing dots/spaces (also invalid on Windows)
 * are stripped; only the EXACT `.`/`..` segments are dropped, so dotfiles like
 * `.env` keep their name; and the UTF-8 encoding is capped at
 * `GFS_ZIP_MAX_SEGMENT_NAME_BYTES` so the 16-bit ZIP name fields can never
 * truncate. Traversal is impossible regardless: separators are gone before the
 * segment is used.
 */
/** Exported for property-based invariant tests (R1-M6). */
export function sanitizeZipSegment(name: string): string {
  const cleaned = truncateToUtf8Bytes(
    name
      .replace(/[/\\:*?"<>|\u0000]/g, '_')
      .replace(/[\s.]+$/, '')
      .trim(),
    GFS_ZIP_MAX_SEGMENT_NAME_BYTES
  )
  if (!cleaned || cleaned === '.' || cleaned === '..') return 'unnamed'
  // The escape happens AFTER the truncation (a truncated name can end up
  // reserved) and is re-trimmed, so the segment still fits the byte ceiling.
  return truncateToUtf8Bytes(
    escapeReservedWindowsDeviceName(cleaned),
    GFS_ZIP_MAX_SEGMENT_NAME_BYTES
  )
}

function sanitizeFileName(name: string): string {
  const cleaned = name
    .replace(/[/\\:*?"<>|\u0000]/g, '_')
    .replace(/[\s.]+$/, '')
    .trim()
  return cleaned || 'folder'
}

/** The saved archive's own file name, escaped from Windows device names. */
function sanitizeArchiveFileName(name: string): string {
  const cleaned = sanitizeFileName(name)
  const separator = cleaned.lastIndexOf('.')
  const stem = separator > 0 ? cleaned.slice(0, separator) : cleaned
  const extension = separator > 0 ? cleaned.slice(separator) : ''
  return `${escapeReservedWindowsDeviceName(stem)}${extension}`
}

export async function createGfsFolderZip(
  folder: GfsZipFolderSource,
  options: CreateGfsFolderZipOptions = {}
): Promise<GfsFolderZipResult> {
  const maxTotalBytes = options.limits?.maxTotalBytes ?? GFS_ZIP_MAX_TOTAL_BYTES
  const maxEntries = options.limits?.maxEntries ?? GFS_ZIP_MAX_ENTRIES
  // Defaults read the real bridge; the walk's AbortSignal rides along on both
  // call kinds so a Stop cancels the producer-side fetch, not just the
  // renderer's patience (R1-M1).
  const listChildren: GfsFolderZipDeps['listChildren'] =
    options.deps?.listChildren ??
    ((resourceId, drive, cursor, requestOptions) =>
      window.clerum.gfs.listChildren(resourceId, drive, cursor, {
        signal: requestOptions?.signal ?? options.signal,
      }))
  // Default lazily reaches the bridge so a walk that never appends (aborted
  // early, refused at planning) does not require window.clerum to exist.
  const zipStream: GfsFolderZipDeps['zipStream'] = options.deps?.zipStream ?? {
    start: () => window.clerum.gfs.zipStream.start(),
    appendFromUri: (jobId, name, uri, requestOptions) =>
      window.clerum.gfs.zipStream.appendFromUri(jobId, name, uri, {
        maxBytes: requestOptions?.maxBytes,
        signal: requestOptions?.signal ?? options.signal,
      }),
    finish: (jobId, suggestedName) => window.clerum.gfs.zipStream.finish(jobId, suggestedName),
    abort: jobId => window.clerum.gfs.zipStream.abort(jobId),
  }
  const sleep =
    options.deps?.sleep ?? ((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)))
  const throttle = options.deps?.throttle ?? createGfsReadThrottle()

  const report = (progress: GfsFolderZipProgress) => options.onProgress?.(progress)

  const signal = options.signal
  const throwIfAborted = () => {
    if (signal?.aborted) throw folderZipAbortedError()
  }
  // Rejects when the caller aborts; raced against every wait so a stop lands
  // immediately instead of after the current throttle slot or backoff. The
  // no-op catch keeps a late abort from becoming an unhandled rejection after
  // the walk already finished.
  const abortRejection = signal
    ? new Promise<never>((_, reject) => {
        signal.addEventListener(
          'abort',
          () => {
            reject(folderZipAbortedError())
          },
          { once: true }
        )
      })
    : null
  if (abortRejection) abortRejection.catch(() => undefined)
  const withAbort = <T>(promise: Promise<T>): Promise<T> =>
    abortRejection ? Promise.race([promise, abortRejection]) : promise

  /**
   * One 429 backs off by the server-provided hint and retries exactly once.
   * The backoff window is NOT a throttle slot: after it elapses the retry
   * re-enters the budget through `throttle.acquire()`, so a
   * `retryAfterSeconds=0` answer cannot fire the retry with zero spacing
   * against the same per-minute budget that just refused the first attempt.
   * Both attempts race the abort signal, so a Stop during the retried request
   * preempts immediately instead of waiting out the in-flight call.
   */
  const withRateLimitRetry = async <T>(operation: () => Promise<T>): Promise<T> => {
    try {
      return await withAbort(operation())
    } catch (error) {
      if (isFolderZipAbortError(error)) throw error
      const message = toMessage(error)
      if (!isRateLimited(message)) throw error
      await withAbort(sleep(Math.min((parseRetryAfterSeconds(message) ?? 15) * 1000, 120_000)))
      await withAbort(throttle.acquire())
      return withAbort(operation())
    }
  }

  const skipped: GfsZipSkippedEntry[] = []
  const files: Array<{ uri: string; path: string }> = []
  const rootName = sanitizeZipSegment(folder.name)
  const queue: Array<{ resourceId: string; prefix: string }> = [
    { resourceId: folder.resourceId, prefix: rootName },
  ]
  const visitedFolders = new Set<string>()
  let plannedBytes = 0
  // Every accepted child, files AND folders, against `maxEntries` (R1-M3).
  let entriesSeen = 0
  // ── Phase 1: recursive listing (breadth-first, cursor-paginated). ──
  report({ phase: 'listing', filesFound: 0, filesAdded: 0, currentPath: rootName })
  while (queue.length) {
    throwIfAborted()
    const next = queue.shift()!
    if (visitedFolders.has(next.resourceId)) continue
    visitedFolders.add(next.resourceId)
    let cursor: string | undefined
    do {
      throwIfAborted()
      await withAbort(throttle.acquire())
      let page: GfsZipChildrenPage
      try {
        page = await withRateLimitRetry(() =>
          listChildren(next.resourceId, folder.drive, cursor, { signal })
        )
      } catch (error) {
        const message = toMessage(error)
        if (isAccessDenied(message)) {
          skipped.push({ path: next.prefix, reason: 'Permission denied' })
          break
        }
        throw error
      }
      for (const child of page.items) {
        const path = `${next.prefix}/${sanitizeZipSegment(child.name)}`
        if (zipPathEncoder.encode(path).length >= GFS_ZIP_MAX_ENTRY_PATH_BYTES) {
          skipped.push({ path: shortenForNotice(path), reason: 'Path too long' })
          continue
        }
        if (child.readable === false) {
          skipped.push({ path, reason: 'No access' })
          continue
        }
        // Directories cost the same walk work as files (one listing request
        // each), so the entry budget counts BOTH kinds (R1-M3) — 2000 empty
        // folders is 2000 requests, not zero work.
        if (entriesSeen + 1 > maxEntries) {
          throw new GfsFolderZipLimitError(
            `"${folder.name}" holds more than ${maxEntries} entries (files and folders), which exceeds the folder-zip limit. Download smaller subfolders individually.`
          )
        }
        entriesSeen += 1
        if (child.kind === 'directory') {
          queue.push({ resourceId: child.resourceId, prefix: path })
          continue
        }
        // Older servers omit `bytes`; a missing value must read as 0, not
        // NaN — NaN would poison `plannedBytes` and silently disable the
        // size guard for the rest of the walk (M1).
        const childBytes =
          typeof child.bytes === 'number' && Number.isFinite(child.bytes) ? child.bytes : 0
        plannedBytes += childBytes
        if (plannedBytes > maxTotalBytes) {
          throw new GfsFolderZipLimitError(
            `"${folder.name}" exceeds the ${formatZipBytes(maxTotalBytes)} folder-zip limit. Download smaller subfolders individually.`
          )
        }
        files.push({ uri: child.gfsUri, path })
        report({
          phase: 'listing',
          filesFound: files.length,
          filesAdded: 0,
          currentPath: path,
        })
      }
      cursor = page.nextCursor ?? undefined
    } while (cursor)
  }

  // An archive with zero entries is never useful: short-circuit instead of
  // saving an empty zip (L8). Any skips are carried on the error so the caller
  // can still explain why nothing was archivable.
  if (files.length === 0) throw new GfsFolderZipEmptyError(folder.name, skipped)

  // ── Phase 2: throttled downloads streamed into the main-process archive. ──
  // The renderer holds NO payload bytes at all (R1-H2): each transfer is
  // fetched and written by the main process, chunk by chunk, straight into the
  // archive's temp file. Any failure aborts the stream job so the temp file
  // never outlives the walk.
  const { jobId } = await zipStream.start()
  let archivedCount = 0
  const nameAllocator = new ZipEntryNameAllocator()
  // Reserve the DIRECTORY paths the archive will imply BEFORE any file name
  // is allocated (R3-M5): a file that shares a name with one of them is
  // renamed, never the directory — and the outcome no longer depends on the
  // order the listing happened to return the entries in.
  for (const directoryPath of impliedDirectoryPaths(files)) {
    nameAllocator.reserveDirectory(directoryPath)
  }
  let addedBytes = 0
  try {
    for (const [index, file] of files.entries()) {
      throwIfAborted()
      // R2-L2 — exact budget exhaustion: stop BEFORE the next transfer so a
      // zero-or-negative bound is never requested; the un-fetched files are
      // reported as visible skips and the archive finishes cleanly with what
      // it holds (spec: round-2 extensions).
      const remainingBytes = maxTotalBytes - addedBytes
      if (remainingBytes <= 0) {
        for (const remainingFile of files.slice(index)) {
          skipped.push({ path: remainingFile.path, reason: 'Zip byte limit reached' })
        }
        break
      }
      // 1-based: the copy says "downloading N of M" while file N is fetched.
      report({
        phase: 'downloading',
        filesFound: files.length,
        filesAdded: index + 1,
        currentPath: file.path,
      })
      await withAbort(throttle.acquire())
      // Finalize the entry name HERE (case-insensitive collision suffix, and
      // the path-component reservation that keeps a file and a directory from
      // sharing a name on a case-insensitive filesystem) and validate the
      // FINALIZED name against the 16-bit ZIP fields before it is sent: a
      // name that only overflows after suffixing is a visible skip, never a
      // truncated header (R1-M2, R3-M5, spec: round-2).
      const plan = nameAllocator.plan(file.path)
      if (!plan.fits) {
        skipped.push({ path: shortenForNotice(file.path), reason: 'Path too long' })
        continue
      }
      let appended: { name: string; bytes: number }
      try {
        // R1-H2: the transfer is bounded producer-side by the remaining
        // budget, so an over-limit body is rejected before it is written.
        appended = await withRateLimitRetry(() =>
          zipStream.appendFromUri(jobId, plan.name, file.uri, {
            maxBytes: remainingBytes,
            signal,
          })
        )
      } catch (error) {
        if (isFolderZipAbortError(error)) throw error
        const message = toMessage(error)
        if (parseHttpStatus(message) === 413) {
          // The remaining budget cannot hold this body: skip it visibly and
          // let the walk terminate cleanly (later files hit remaining-0).
          skipped.push({ path: file.path, reason: 'Zip byte limit reached' })
          continue
        }
        if (isAccessDenied(message)) {
          skipped.push({ path: file.path, reason: 'Permission denied' })
          continue
        }
        throw error
      }
      nameAllocator.commit(plan)
      addedBytes += appended.bytes
      if (addedBytes > maxTotalBytes) {
        // Defensive backstop: the producer-side bound makes this unreachable
        // unless the bound itself was ignored — fail loudly, not silently.
        throw new GfsFolderZipLimitError(
          `"${folder.name}" exceeded the ${formatZipBytes(maxTotalBytes)} folder-zip limit while downloading. Download smaller subfolders individually.`
        )
      }
      archivedCount += 1
    }

    // Downloads can skip every file (permissions, exhausted budget): the
    // empty short-circuit applies to the archived result, not only the
    // planned list.
    if (archivedCount === 0) throw new GfsFolderZipEmptyError(folder.name, skipped)

    // ── Phase 3: finalize and save through the native dialog. ──
    report({
      phase: 'assembling',
      filesFound: files.length,
      filesAdded: archivedCount,
      currentPath: null,
    })
    const fileName = `${sanitizeArchiveFileName(folder.name)}.zip`
    const saved = await zipStream.finish(jobId, fileName)
    return {
      saved: saved.saved,
      savedPath: saved.filePath,
      fileName,
      fileCount: saved.entryCount,
      skipped,
    }
  } catch (error) {
    // Any failure (abort included) must not leave a temp file behind.
    await zipStream.abort(jobId).catch(() => undefined)
    throw error
  }
}

export { foldEntryName }

/**
 * Every directory path the archive will imply, in first-seen order: each
 * file's path minus its last segment, and every ancestor of that.
 */
function impliedDirectoryPaths(files: Array<{ path: string }>): string[] {
  const paths = new Set<string>()
  for (const file of files) {
    const segments = file.path.split('/')
    segments.pop()
    let current = ''
    for (const segment of segments) {
      current = current === '' ? segment : `${current}/${segment}`
      paths.add(current)
    }
  }
  return [...paths]
}

function formatZipBytes(bytes: number): string {
  if (bytes >= 1024 * 1024 * 1024) return `${Math.round(bytes / (1024 * 1024 * 1024))} GiB`
  if (bytes >= 1024 * 1024) return `${Math.round(bytes / (1024 * 1024))} MiB`
  return `${Math.round(bytes / 1024)} KiB`
}

/** Visible skip notice for the result toast: names the first few skips. */
export function describeZipSkips(skipped: GfsZipSkippedEntry[]): string {
  const visible = skipped.slice(0, 3).map(entry => `${entry.path} (${entry.reason})`)
  const more = skipped.length - visible.length
  const names = more > 0 ? `${visible.join(', ')} and ${more} more` : visible.join(', ')
  return `Skipped ${skipped.length} ${skipped.length === 1 ? 'entry' : 'entries'}: ${names}`
}