Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* BUG-175 — recursive "Download as zip" for a GFS folder. The walk runs in the
* renderer over the existing `window.clerum.gfs` read plane; the ARCHIVE is
* assembled streamed in the main process (temp file + native save dialog, see
* gfs/zipStream.ts), so the renderer never holds archive bytes and its peak
* memory is one in-flight transfer, independent of folder size (R1-H2, spec:
* round-2 extensions).
*
* Budget contract: the producer meters this walk's request classes per actor —
* resource reads (children, affordances, resolve) and proxy reads (downloads)
* — at 480/min each by default (control-api `externalGfsResourceReadRlPerMin`
* / `externalGfsProxyReadRlPerMin`). Every listing and download below goes
* through `createGfsReadThrottle`, deliberately spaced at a CONSERVATIVE
* fraction of those ceilings (see `GFS_ZIP_READS_PER_MINUTE`) so one walk
* leaves room for the browsing surfaces sharing the same classes, and a 429
* answer backs off once by the server-provided `retryAfterSeconds` before
* failing. Byte and entry ceilings bound the walk before anything large is
* buffered, and entries the caller cannot read (a listing row marked
* unreadable, or a 403 on the download itself) are skipped and reported back
* so the result can show a visible skip notice instead of a silently holey
* archive.
*/
import { isRateLimited, parseHttpStatus, parseRetryAfterSeconds } from '@lib/gfsGrantErrors'
import { ZipEntryNameAllocator, foldEntryName } from '../../../src/gfs/zipEntryName'
import { escapeReservedWindowsDeviceName } from '../../../src/gfs/zipEntryPath'
import { ZIP_MAX_ENTRIES, ZIP_MAX_TOTAL_BYTES } from '../../../src/gfs/zipLimits'
/**
* Client-side pacing for the walk, a deliberate conservative margin: the
* producer's default ceilings for the classes used here are 480/min each
* (resource reads, proxy reads); spacing at 120/min keeps one walk well under
* them while the Files browsing surface shares the same classes.
*/
export const GFS_ZIP_READS_PER_MINUTE = 120
/**
* Refuse before buffering: total bytes across the folder. 512 MiB bounds the
* streamed archive's temp-file size and each transfer's producer-side bound;
* the RENDERER holds no payload bytes at all (the main process fetches and
* writes each transfer chunk by chunk), so peak memory is one chunk in main
* plus the archive's central-directory metadata (spec: round-2 extensions,
* streamed assembly; R1-H2).
*/
export const GFS_ZIP_MAX_TOTAL_BYTES = ZIP_MAX_TOTAL_BYTES
/** Refuse before buffering: total file entries across the folder. */
export const GFS_ZIP_MAX_ENTRIES = ZIP_MAX_ENTRIES
export interface GfsZipSkippedEntry {
path: string
reason: string
}
export type GfsFolderZipPhase = 'listing' | 'downloading' | 'assembling'
export interface GfsFolderZipProgress {
phase: GfsFolderZipPhase
filesFound: number
filesAdded: number
currentPath: string | null
}
export interface GfsFolderZipResult {
/** False when the user canceled the native save dialog (temp file deleted). */
saved: boolean
savedPath: string | null
fileName: string
fileCount: number
skipped: GfsZipSkippedEntry[]
}
/** The walk exceeded a configured ceiling — a refusal, not a failure. */
export class GfsFolderZipLimitError extends Error {}
/** Nothing was archivable (an empty folder, or every entry was skipped). */
export class GfsFolderZipEmptyError extends Error {
readonly skipped: GfsZipSkippedEntry[]
constructor(folderName: string, skipped: GfsZipSkippedEntry[]) {
super(`"${folderName}" has no downloadable files.`)
this.name = 'GfsFolderZipEmptyError'
this.skipped = skipped
}
}
export interface GfsReadThrottle {
acquire(): Promise<void>
}
export interface CreateGfsReadThrottleOptions {
requestsPerMinute?: number
now?: () => number
sleep?: (ms: number) => Promise<void>
}
/**
* Sequential read spacing for the server's per-actor GFS budget: one acquired
* slot per request, `60_000 / requestsPerMinute` apart. Injectable clock and
* sleep keep the spacing testable without real time.
*/
export function createGfsReadThrottle(options: CreateGfsReadThrottleOptions = {}): GfsReadThrottle {
const requestsPerMinute = options.requestsPerMinute ?? GFS_ZIP_READS_PER_MINUTE
const now = options.now ?? (() => Date.now())
const sleep = options.sleep ?? (ms => new Promise<void>(resolve => setTimeout(resolve, ms)))
const intervalMs = 60_000 / requestsPerMinute
let nextAllowedAt = 0
return {
async acquire() {
const current = now()
if (current < nextAllowedAt) {
await sleep(nextAllowedAt - current)
}
nextAllowedAt = Math.max(nextAllowedAt, now()) + intervalMs
},
}
}
export interface GfsZipChildItem {
resourceId: string
gfsUri: string
name: string
kind: 'file' | 'directory'
/** Absent on older servers — treated as 0 by the size guard, never as NaN. */
bytes?: number
drive?: string
readable?: boolean
}
export interface GfsZipChildrenPage {
items: GfsZipChildItem[]
nextCursor: string | null
}
export interface GfsZipStreamDeps {
start(): Promise<{ jobId: string }>
/**
* Appends one entry by handing the MAIN process the resource URI: the
* payload is fetched and written into the archive chunk by chunk, so it
* never crosses the IPC boundary and never materializes as one buffer
* (R1-H2). `options.maxBytes` is the producer-side bound for this
* transfer; `options.signal` stops the producer-side fetch.
*/
appendFromUri(
jobId: string,
name: string,
uri: string,
options?: { maxBytes?: number; signal?: AbortSignal }
): Promise<{ name: string; bytes: number }>
finish(
jobId: string,
suggestedName: string
): Promise<{ saved: boolean; filePath: string | null; entryCount: number }>
abort(jobId: string): Promise<unknown>
}
export interface GfsFolderZipDeps {
/**
* Lists one folder page. `options.signal` propagates the walk's stop to the
* producer-side fetch when the default bridge is used (R1-M1).
*/
listChildren(
resourceId: string,
drive: string,
cursor?: string,
options?: { signal?: AbortSignal }
): Promise<GfsZipChildrenPage>
/**
* Streamed, file-backed archive assembly in the main process (R1-H2): one
* append per downloaded file, one finish at the end, abort cleans the temp
* file on any failure path.
*/
zipStream: GfsZipStreamDeps
throttle: GfsReadThrottle
sleep: (ms: number) => Promise<void>
}
export interface CreateGfsFolderZipOptions {
onProgress?: (progress: GfsFolderZipProgress) => void
limits?: { maxTotalBytes?: number; maxEntries?: number }
deps?: Partial<GfsFolderZipDeps>
/**
* Cooperative stop for a walk that can legally run for many minutes: every
* wait (throttle slot, 429 backoff, in-flight request) races the signal, and
* each loop turn re-checks it. Aborting rejects with a DOMException named
* `AbortError` (`isFolderZipAbortError`).
*/
signal?: AbortSignal
}
export interface GfsZipFolderSource {
resourceId: string
drive: string
name: string
}
/**
* Complete-entry path bound. ZIP name fields are 16-bit BYTE counts, so the
* fully assembled entry path (all segments joined) must stay strictly under
* 65535 encoded bytes. Segments are capped individually
* (`GFS_ZIP_MAX_SEGMENT_NAME_BYTES`); this bound closes the deep-nesting case
* where many legal segments stack past the field. An overlong entry is skipped
* with a visible notice, and an overlong DIRECTORY prunes its subtree — every
* descendant would overflow too (spec: ZIP budget model).
*/
export const GFS_ZIP_MAX_ENTRY_PATH_BYTES = 65535
const zipPathEncoder = new TextEncoder()
/** Human-noticeable stand-in for an over-long path (the real one is unusable). */
function shortenForNotice(path: string): string {
return path.length > 80 ? `${path.slice(0, 48)}…${path.slice(-24)}` : path
}
function toMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error)
}
/** The user stopped the walk — not a failure, and never a save. */
export function isFolderZipAbortError(error: unknown): boolean {
return error instanceof DOMException && error.name === 'AbortError'
}
function folderZipAbortedError(): DOMException {
return new DOMException('Folder-zip walk was stopped.', 'AbortError')
}
/** A per-resource denial for THIS entry (403/forbidden), not a session failure. */
function isAccessDenied(message: string): boolean {
return parseHttpStatus(message) === 403 || message.toLowerCase().includes('forbidden')
}
/**
* Zip entry-name length ceiling per segment. ZIP name fields are 16-bit byte
* counts; a name whose UTF-8 encoding reaches 65 536 bytes would silently
* truncate there and corrupt the archive, so segments are cut well under it.
*/
export const GFS_ZIP_MAX_SEGMENT_NAME_BYTES = 1024
const segmentNameEncoder = new TextEncoder()
/** Truncates by code point so the UTF-8 encoding stays `maxBytes` or less. */
function truncateToUtf8Bytes(name: string, maxBytes: number): string {
if (segmentNameEncoder.encode(name).length <= maxBytes) return name
let kept = ''
let used = 0
for (const character of name) {
const size = segmentNameEncoder.encode(character).length
if (used + size > maxBytes) break
kept += character
used += size
}
return kept
}
/**
* Zip-safe path segment. Separators, NUL and the characters Windows forbids
* (`: * ? " < > |`) map to `_`; trailing dots/spaces (also invalid on Windows)
* are stripped; only the EXACT `.`/`..` segments are dropped, so dotfiles like
* `.env` keep their name; and the UTF-8 encoding is capped at
* `GFS_ZIP_MAX_SEGMENT_NAME_BYTES` so the 16-bit ZIP name fields can never
* truncate. Traversal is impossible regardless: separators are gone before the
* segment is used.
*/
/** Exported for property-based invariant tests (R1-M6). */
export function sanitizeZipSegment(name: string): string {
const cleaned = truncateToUtf8Bytes(
name
.replace(/[/\\:*?"<>|\u0000]/g, '_')
.replace(/[\s.]+$/, '')
.trim(),
GFS_ZIP_MAX_SEGMENT_NAME_BYTES
)
if (!cleaned || cleaned === '.' || cleaned === '..') return 'unnamed'
// The escape happens AFTER the truncation (a truncated name can end up
// reserved) and is re-trimmed, so the segment still fits the byte ceiling.
return truncateToUtf8Bytes(
escapeReservedWindowsDeviceName(cleaned),
GFS_ZIP_MAX_SEGMENT_NAME_BYTES
)
}
function sanitizeFileName(name: string): string {
const cleaned = name
.replace(/[/\\:*?"<>|\u0000]/g, '_')
.replace(/[\s.]+$/, '')
.trim()
return cleaned || 'folder'
}
/** The saved archive's own file name, escaped from Windows device names. */
function sanitizeArchiveFileName(name: string): string {
const cleaned = sanitizeFileName(name)
const separator = cleaned.lastIndexOf('.')
const stem = separator > 0 ? cleaned.slice(0, separator) : cleaned
const extension = separator > 0 ? cleaned.slice(separator) : ''
return `${escapeReservedWindowsDeviceName(stem)}${extension}`
}
export async function createGfsFolderZip(
folder: GfsZipFolderSource,
options: CreateGfsFolderZipOptions = {}
): Promise<GfsFolderZipResult> {
const maxTotalBytes = options.limits?.maxTotalBytes ?? GFS_ZIP_MAX_TOTAL_BYTES
const maxEntries = options.limits?.maxEntries ?? GFS_ZIP_MAX_ENTRIES
// Defaults read the real bridge; the walk's AbortSignal rides along on both
// call kinds so a Stop cancels the producer-side fetch, not just the
// renderer's patience (R1-M1).
const listChildren: GfsFolderZipDeps['listChildren'] =
options.deps?.listChildren ??
((resourceId, drive, cursor, requestOptions) =>
window.clerum.gfs.listChildren(resourceId, drive, cursor, {
signal: requestOptions?.signal ?? options.signal,
}))
// Default lazily reaches the bridge so a walk that never appends (aborted
// early, refused at planning) does not require window.clerum to exist.
const zipStream: GfsFolderZipDeps['zipStream'] = options.deps?.zipStream ?? {
start: () => window.clerum.gfs.zipStream.start(),
appendFromUri: (jobId, name, uri, requestOptions) =>
window.clerum.gfs.zipStream.appendFromUri(jobId, name, uri, {
maxBytes: requestOptions?.maxBytes,
signal: requestOptions?.signal ?? options.signal,
}),
finish: (jobId, suggestedName) => window.clerum.gfs.zipStream.finish(jobId, suggestedName),
abort: jobId => window.clerum.gfs.zipStream.abort(jobId),
}
const sleep =
options.deps?.sleep ?? ((ms: number) => new Promise<void>(resolve => setTimeout(resolve, ms)))
const throttle = options.deps?.throttle ?? createGfsReadThrottle()
const report = (progress: GfsFolderZipProgress) => options.onProgress?.(progress)
const signal = options.signal
const throwIfAborted = () => {
if (signal?.aborted) throw folderZipAbortedError()
}
// Rejects when the caller aborts; raced against every wait so a stop lands
// immediately instead of after the current throttle slot or backoff. The
// no-op catch keeps a late abort from becoming an unhandled rejection after
// the walk already finished.
const abortRejection = signal
? new Promise<never>((_, reject) => {
signal.addEventListener(
'abort',
() => {
reject(folderZipAbortedError())
},
{ once: true }
)
})
: null
if (abortRejection) abortRejection.catch(() => undefined)
const withAbort = <T>(promise: Promise<T>): Promise<T> =>
abortRejection ? Promise.race([promise, abortRejection]) : promise
/**
* One 429 backs off by the server-provided hint and retries exactly once.
* The backoff window is NOT a throttle slot: after it elapses the retry
* re-enters the budget through `throttle.acquire()`, so a
* `retryAfterSeconds=0` answer cannot fire the retry with zero spacing
* against the same per-minute budget that just refused the first attempt.
* Both attempts race the abort signal, so a Stop during the retried request
* preempts immediately instead of waiting out the in-flight call.
*/
const withRateLimitRetry = async <T>(operation: () => Promise<T>): Promise<T> => {
try {
return await withAbort(operation())
} catch (error) {
if (isFolderZipAbortError(error)) throw error
const message = toMessage(error)
if (!isRateLimited(message)) throw error
await withAbort(sleep(Math.min((parseRetryAfterSeconds(message) ?? 15) * 1000, 120_000)))
await withAbort(throttle.acquire())
return withAbort(operation())
}
}
const skipped: GfsZipSkippedEntry[] = []
const files: Array<{ uri: string; path: string }> = []
const rootName = sanitizeZipSegment(folder.name)
const queue: Array<{ resourceId: string; prefix: string }> = [
{ resourceId: folder.resourceId, prefix: rootName },
]
const visitedFolders = new Set<string>()
let plannedBytes = 0
// Every accepted child, files AND folders, against `maxEntries` (R1-M3).
let entriesSeen = 0
// ── Phase 1: recursive listing (breadth-first, cursor-paginated). ──
report({ phase: 'listing', filesFound: 0, filesAdded: 0, currentPath: rootName })
while (queue.length) {
throwIfAborted()
const next = queue.shift()!
if (visitedFolders.has(next.resourceId)) continue
visitedFolders.add(next.resourceId)
let cursor: string | undefined
do {
throwIfAborted()
await withAbort(throttle.acquire())
let page: GfsZipChildrenPage
try {
page = await withRateLimitRetry(() =>
listChildren(next.resourceId, folder.drive, cursor, { signal })
)
} catch (error) {
const message = toMessage(error)
if (isAccessDenied(message)) {
skipped.push({ path: next.prefix, reason: 'Permission denied' })
break
}
throw error
}
for (const child of page.items) {
const path = `${next.prefix}/${sanitizeZipSegment(child.name)}`
if (zipPathEncoder.encode(path).length >= GFS_ZIP_MAX_ENTRY_PATH_BYTES) {
skipped.push({ path: shortenForNotice(path), reason: 'Path too long' })
continue
}
if (child.readable === false) {
skipped.push({ path, reason: 'No access' })
continue
}
// Directories cost the same walk work as files (one listing request
// each), so the entry budget counts BOTH kinds (R1-M3) — 2000 empty
// folders is 2000 requests, not zero work.
if (entriesSeen + 1 > maxEntries) {
throw new GfsFolderZipLimitError(
`"${folder.name}" holds more than ${maxEntries} entries (files and folders), which exceeds the folder-zip limit. Download smaller subfolders individually.`
)
}
entriesSeen += 1
if (child.kind === 'directory') {
queue.push({ resourceId: child.resourceId, prefix: path })
continue
}
// Older servers omit `bytes`; a missing value must read as 0, not
// NaN — NaN would poison `plannedBytes` and silently disable the
// size guard for the rest of the walk (M1).
const childBytes =
typeof child.bytes === 'number' && Number.isFinite(child.bytes) ? child.bytes : 0
plannedBytes += childBytes
if (plannedBytes > maxTotalBytes) {
throw new GfsFolderZipLimitError(
`"${folder.name}" exceeds the ${formatZipBytes(maxTotalBytes)} folder-zip limit. Download smaller subfolders individually.`
)
}
files.push({ uri: child.gfsUri, path })
report({
phase: 'listing',
filesFound: files.length,
filesAdded: 0,
currentPath: path,
})
}
cursor = page.nextCursor ?? undefined
} while (cursor)
}
// An archive with zero entries is never useful: short-circuit instead of
// saving an empty zip (L8). Any skips are carried on the error so the caller
// can still explain why nothing was archivable.
if (files.length === 0) throw new GfsFolderZipEmptyError(folder.name, skipped)
// ── Phase 2: throttled downloads streamed into the main-process archive. ──
// The renderer holds NO payload bytes at all (R1-H2): each transfer is
// fetched and written by the main process, chunk by chunk, straight into the
// archive's temp file. Any failure aborts the stream job so the temp file
// never outlives the walk.
const { jobId } = await zipStream.start()
let archivedCount = 0
const nameAllocator = new ZipEntryNameAllocator()
// Reserve the DIRECTORY paths the archive will imply BEFORE any file name
// is allocated (R3-M5): a file that shares a name with one of them is
// renamed, never the directory — and the outcome no longer depends on the
// order the listing happened to return the entries in.
for (const directoryPath of impliedDirectoryPaths(files)) {
nameAllocator.reserveDirectory(directoryPath)
}
let addedBytes = 0
try {
for (const [index, file] of files.entries()) {
throwIfAborted()
// R2-L2 — exact budget exhaustion: stop BEFORE the next transfer so a
// zero-or-negative bound is never requested; the un-fetched files are
// reported as visible skips and the archive finishes cleanly with what
// it holds (spec: round-2 extensions).
const remainingBytes = maxTotalBytes - addedBytes
if (remainingBytes <= 0) {
for (const remainingFile of files.slice(index)) {
skipped.push({ path: remainingFile.path, reason: 'Zip byte limit reached' })
}
break
}
// 1-based: the copy says "downloading N of M" while file N is fetched.
report({
phase: 'downloading',
filesFound: files.length,
filesAdded: index + 1,
currentPath: file.path,
})
await withAbort(throttle.acquire())
// Finalize the entry name HERE (case-insensitive collision suffix, and
// the path-component reservation that keeps a file and a directory from
// sharing a name on a case-insensitive filesystem) and validate the
// FINALIZED name against the 16-bit ZIP fields before it is sent: a
// name that only overflows after suffixing is a visible skip, never a
// truncated header (R1-M2, R3-M5, spec: round-2).
const plan = nameAllocator.plan(file.path)
if (!plan.fits) {
skipped.push({ path: shortenForNotice(file.path), reason: 'Path too long' })
continue
}
let appended: { name: string; bytes: number }
try {
// R1-H2: the transfer is bounded producer-side by the remaining
// budget, so an over-limit body is rejected before it is written.
appended = await withRateLimitRetry(() =>
zipStream.appendFromUri(jobId, plan.name, file.uri, {
maxBytes: remainingBytes,
signal,
})
)
} catch (error) {
if (isFolderZipAbortError(error)) throw error
const message = toMessage(error)
if (parseHttpStatus(message) === 413) {
// The remaining budget cannot hold this body: skip it visibly and
// let the walk terminate cleanly (later files hit remaining-0).
skipped.push({ path: file.path, reason: 'Zip byte limit reached' })
continue
}
if (isAccessDenied(message)) {
skipped.push({ path: file.path, reason: 'Permission denied' })
continue
}
throw error
}
nameAllocator.commit(plan)
addedBytes += appended.bytes
if (addedBytes > maxTotalBytes) {
// Defensive backstop: the producer-side bound makes this unreachable
// unless the bound itself was ignored — fail loudly, not silently.
throw new GfsFolderZipLimitError(
`"${folder.name}" exceeded the ${formatZipBytes(maxTotalBytes)} folder-zip limit while downloading. Download smaller subfolders individually.`
)
}
archivedCount += 1
}
// Downloads can skip every file (permissions, exhausted budget): the
// empty short-circuit applies to the archived result, not only the
// planned list.
if (archivedCount === 0) throw new GfsFolderZipEmptyError(folder.name, skipped)
// ── Phase 3: finalize and save through the native dialog. ──
report({
phase: 'assembling',
filesFound: files.length,
filesAdded: archivedCount,
currentPath: null,
})
const fileName = `${sanitizeArchiveFileName(folder.name)}.zip`
const saved = await zipStream.finish(jobId, fileName)
return {
saved: saved.saved,
savedPath: saved.filePath,
fileName,
fileCount: saved.entryCount,
skipped,
}
} catch (error) {
// Any failure (abort included) must not leave a temp file behind.
await zipStream.abort(jobId).catch(() => undefined)
throw error
}
}
export { foldEntryName }
/**
* Every directory path the archive will imply, in first-seen order: each
* file's path minus its last segment, and every ancestor of that.
*/
function impliedDirectoryPaths(files: Array<{ path: string }>): string[] {
const paths = new Set<string>()
for (const file of files) {
const segments = file.path.split('/')
segments.pop()
let current = ''
for (const segment of segments) {
current = current === '' ? segment : `${current}/${segment}`
paths.add(current)
}
}
return [...paths]
}
function formatZipBytes(bytes: number): string {
if (bytes >= 1024 * 1024 * 1024) return `${Math.round(bytes / (1024 * 1024 * 1024))} GiB`
if (bytes >= 1024 * 1024) return `${Math.round(bytes / (1024 * 1024))} MiB`
return `${Math.round(bytes / 1024)} KiB`
}
/** Visible skip notice for the result toast: names the first few skips. */
export function describeZipSkips(skipped: GfsZipSkippedEntry[]): string {
const visible = skipped.slice(0, 3).map(entry => `${entry.path} (${entry.reason})`)
const more = skipped.length - visible.length
const names = more > 0 ? `${visible.join(', ')} and ${more} more` : visible.join(', ')
return `Skipped ${skipped.length} ${skipped.length === 1 ? 'entry' : 'entries'}: ${names}`
}