The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import { mkdir, statfs as fsStatfs } from 'node:fs/promises'
import { env } from '../config/env.js'
import { copyPasteDropFile, PASTE_DROPS_DIR, writePasteDropStream } from '../lib/pasteDropFiles.js'
import { object } from './wire.js'
/** The most a file sent by the multipart route may be (lib/superTabAttachmentProxy.ts); a larger one goes by a link. */
export { MAX_FORM_ATTACHMENT_BYTES } from '../lib/superTabAttachmentProxy.js'
/** As the backend hands it with a message (backend/src/superTabs/attachments.ts): `link` is a signed link of that read, which
* opens until `expiresAt` (epoch ms); `unlinked` says the server could not sign one just now; `url` is the file's public
* address, which stops opening once the bucket is private. The link is used whenever there is one, a fresh one is asked for
* an unlinked file, and the address only for a file with neither, which only an older backend sends. */
export interface MessageAttachment { id: string; name: string; mime: string; size: number; url: string; link?: string; expiresAt?: number
unlinked?: true }
/** A fresh link to [file], from the backend, for whoever this host delivers to (superTabs/host.ts): the read checks that they
* may still read the message. It throws when they may not, or the file is gone. */
export type Relink = (file: MessageAttachment) => Promise<MessageAttachment>
/** A link this close to its end is fetched afresh before a try starts: storage checks it when a request starts, and a
* download that starts in time finishes however long it takes. */
const RELINK_MARGIN_MS = 30_000
/** The backend's per-file limit. A download never reads past it, whatever a message declares. */
export const MAX_ATTACHMENT_BYTES = 256 * 1024 * 1024
/** No bytes for this long ends a download: a slow link keeps going, a stalled one does not hold the host's loop. */
const IDLE_MS = 30_000
/** The slowest steady pace a download is given time for: 256 KB/s, so 256 MB in about 17 minutes. */
const FLOOR_BYTES_PER_MS = 256 * 1024 / 1000
/** One download's ceiling however steadily it moves: half a minute, and its bytes at the slowest pace. */
const downloadMs = (bytes: number) => 30_000 + Math.ceil(bytes / FLOOR_BYTES_PER_MS)
/** A message's time for all its files, which download together and share the link: at least 45 s, and their bytes at the
* slowest pace, so a 256 MB file can finish. */
const BUDGET_MS = 45_000
const budgetMs = (files: MessageAttachment[]) =>
BUDGET_MS + Math.ceil(files.reduce((sum, file) => sum + Math.min(file.size, MAX_ATTACHMENT_BYTES), 0) / FLOOR_BYTES_PER_MS)
const megabytes = (bytes: number) => `${bytes / 1024 / 1024} MB`
/** Kept free beside a download: a 256 MB file must not fill the disk this computer's agents work on. */
const FREE_MARGIN_BYTES = 512 * 1024 * 1024
/** What `fs.statfs` tells of the disk: free blocks for this user, and their size. */
export type Statfs = (path: string) => Promise<{ bavail: number | bigint; bsize: number | bigint }>
/** Whether the paste-drops disk has room for [bytes] and the margin. A disk that cannot say (statfs unsupported) is not refused. */
async function roomFor(bytes: number, statfs: Statfs): Promise<boolean> {
let free: number
try {
await mkdir(PASTE_DROPS_DIR, { recursive: true })
const disk = await statfs(PASTE_DROPS_DIR)
free = Number(disk.bavail) * Number(disk.bsize)
} catch { return true }
return free >= bytes + FREE_MARGIN_BYTES
}
export function attachmentsOf(message: Record<string, unknown>): MessageAttachment[] {
if (!Array.isArray(message.attachments)) return []
return message.attachments.slice(0, 10).map(object).filter((a): a is Record<string, unknown> & MessageAttachment =>
typeof a.name === 'string' && typeof a.url === 'string' && typeof a.mime === 'string'
&& Number.isSafeInteger(a.size) && (a.size as number) >= 0)
.map(a => ({ id: typeof a.id === 'string' ? a.id : '', name: a.name, mime: a.mime, size: a.size, url: a.url,
...typeof a.link === 'string' && a.link ? { link: a.link,
...typeof a.expiresAt === 'number' && Number.isFinite(a.expiresAt) ? { expiresAt: a.expiresAt } : {} }
: a.unlinked === true ? { unlinked: true as const } : {} }))
}
export const formatSize = (bytes: number) => bytes < 1024 ? `${bytes} B`
: bytes < 1024 * 1024 ? `${(bytes / 1024).toFixed(1)} KB` : `${(bytes / 1024 / 1024).toFixed(1)} MB`
/** Only files on Harness's own file storage, under the backend's public base (an older backend) or on the signed-link host:
* a message's URL must not make this computer fetch another address. A signed link must also name the very object its file's
* public address names (sameObject). */
export function servedFrom(base: string, address: string, signedBase = env.SUPER_TAB_FILES_BASE): boolean {
return under(base, address) || under(signedBase, address)
}
function under(base: string, address: string): boolean {
try {
const root = new URL(base), target = new URL(address)
const prefix = root.pathname.endsWith('/') ? root.pathname : `${root.pathname}/`
return target.origin === root.origin && target.pathname.startsWith(prefix) && !target.username && !target.password
} catch { return false }
}
/** A failure that another try cannot fix. */
class Refused extends Error {}
/** What a person and their agent read when the disk is full: before a download, or as one is written or copied (task #3159,
* Scholes's Mac). The system's own error names the files' paths, in the person's home, and is no plainer. */
const NO_ROOM = 'not enough free disk space'
const isNoRoom = (error: unknown) => !!error && typeof error === 'object' && 'code' in error && (error.code === 'ENOSPC' || error.code === 'EDQUOT')
/** The name on disk: the prompt lists this path for an agent that may paste it into a shell, so it carries nothing a
* shell reads as syntax ($(...), quotes, spaces, ;). The original name stays in the label beside it. */
export const diskName = (name: string) => name.replaceAll(/[^A-Za-z0-9._-]+/g, '_').replace(/^[.-]+/, '').slice(-120) || 'file'
/** Whether signed [link] names the same object as public [url]: `/<one bucket>/<the key url names under base>`. Without the
* bucket's name, this binds a link to the one file Harness stored for the message, in a single bucket, on the signed-link
* host (review 1 of #2460); `SUPER_TAB_FILES_BASE` may also name the bucket itself. */
export function sameObject(base: string, url: string, link: string): boolean {
try {
const root = new URL(base), stored = new URL(url), signed = new URL(link)
const prefix = root.pathname.endsWith('/') ? root.pathname : `${root.pathname}/`
if (stored.origin !== root.origin || !stored.pathname.startsWith(prefix)) return false
const key = decodeURIComponent(stored.pathname.slice(prefix.length))
const [blank, bucket, ...rest] = signed.pathname.split('/')
return blank === '' && !!bucket && !!key && rest.map(decodeURIComponent).join('/') === key
} catch { return false }
}
/** Storage's answer to a signed link that is over, or a file the server could not sign: the next try asks for a fresh one
* (attachedFiles). */
class Expired extends Error {}
async function download(file: MessageAttachment, base: string, signedBase: string, fetcher: typeof fetch, ms: number, idleMs: number,
stop: AbortSignal | undefined, statfs: Statfs): Promise<string> {
// An unlinked file is never opened by its public address: once the bucket is private that is a dead page.
if (!file.link && file.unlinked) throw new Expired('not signed just now')
const address = file.link ?? file.url
if (!address) throw new Refused('no longer available')
if (!servedFrom(base, address, signedBase) || (file.link && !sameObject(base, file.url, file.link))) throw new Refused('not a Harness file address')
if (file.size > MAX_ATTACHMENT_BYTES) throw new Refused(`over the ${megabytes(MAX_ATTACHMENT_BYTES)} limit`)
// Another try would find the same disk: refused, not tried again.
if (!await roomFor(file.size, statfs)) throw new Refused(NO_ROOM)
const limit = file.size
// No bytes for idleMs ends it, from the request on: waiting for the answer counts too.
const idle = new AbortController()
let quiet: NodeJS.Timeout | undefined
const moved = () => {
clearTimeout(quiet)
quiet = setTimeout(() => idle.abort(new Error(`no bytes came for ${Math.round(idleMs / 1000)} s`)), idleMs)
}
moved()
try {
// A redirect could leave the public base, so it is a failure, not followed.
const response = await fetcher(address, { redirect: 'error', signal: AbortSignal.any([idle.signal, AbortSignal.timeout(ms), ...stop ? [stop] : []]) })
if (!response.ok || !response.body) {
await response.body?.cancel().catch(() => {})
// A signed link that is over, or whose file is no longer readable through it: a fresh link may still open it.
if ((response.status === 400 || response.status === 403) && file.link) throw new Expired(`HTTP ${response.status}`)
const Failure = response.status >= 400 && response.status < 500 && ![408, 429].includes(response.status) ? Refused : Error
throw new Failure(`HTTP ${response.status}`)
}
// A length it declares past the size is refused before a byte is read. Compressed, the length is not the file's.
const length = response.headers.get('content-length')
if (length !== null && !response.headers.get('content-encoding') && Number(length) > limit) {
await response.body.cancel().catch(() => {})
throw new Refused('larger than declared')
}
const body = response.body
// Counted as it goes to disk: one byte past the size ends it, and so does a body that ends short. Either way the file
// written so far is removed (lib/pasteDropFiles.ts).
async function* counted(): AsyncGenerator<Uint8Array> {
let total = 0
for await (const chunk of body) {
moved()
total += chunk.byteLength
if (total > limit) throw new Refused('larger than declared')
yield chunk
}
if (total !== file.size) throw new Error('incomplete download')
}
return await writePasteDropStream(diskName(file.name), counted())
} catch (error) {
// The disk filled up as it was written: another try finds the same disk. The part written is gone (lib/pasteDropFiles.ts).
if (isNoRoom(error)) throw new Refused(NO_ROOM)
// Ended for want of bytes: said so, rather than as the abort the fetch or the body threw.
throw idle.signal.aborted && !(error instanceof Refused) ? idle.signal.reason : error
} finally { clearTimeout(quiet) }
}
/** The prompt's list of a message's files, downloaded into this computer's paste-drops folder (kept 24 h), so an
* agent on any machine can read them. Each file is streamed to disk, and gets up to three tries within one time budget for
* the message, which grows with its bytes; the host's single poll loop waits on it. A try whose link is over or about to be
* starts with a fresh one from [relink]. A file that could not be downloaded is named by its name and size alone (a signed
* link carries a credential, and is never written into a prompt), and `complete` is false, so the host tries again, with a
* fresh claim and fresh links, before it next delivers the same message. `paths` are the files written, which a Buddy's
* guard may list when its own person sent them (lib/buddyGuard.ts). */
/** A file being downloaded, by its address and size: a second delivery of the same file waits on the first's download
* rather than start its own (superTabs/host.ts keeps one such map). [wanting]: the deliveries still waiting for it; the last
* to leave stops it. */
export type FilesInFlight = Map<string, SharedDownload>
interface SharedDownload { result: Promise<{ line: string; path: string | null }>; wanting: number; stop: AbortController }
export async function attachedFiles(message: Record<string, unknown>,
options: { base?: string; signedBase?: string; fetch?: typeof fetch; budgetMs?: number; idleMs?: number; now?: () => number
inFlight?: FilesInFlight; signal?: AbortSignal; statfs?: Statfs; relink?: Relink } = {}):
Promise<{ text: string; complete: boolean; paths: string[] }> {
const files = attachmentsOf(message)
if (!files.length) return { text: '', complete: true, paths: [] }
const base = options.base ?? env.SUPER_TAB_ATTACHMENTS_PUBLIC_BASE, signedBase = options.signedBase ?? env.SUPER_TAB_FILES_BASE
const now = options.now ?? Date.now, deadline = now() + (options.budgetMs ?? budgetMs(files))
// The sender's name, quoted so it reads as a name and is never taken for the path, which is the shell-safe one.
const labelOf = (file: MessageAttachment) => `sent as ${JSON.stringify(file.name)}, ${formatSize(file.size)}`
const failed = (file: MessageAttachment, why: string) => ({ path: null, line: `- Could not download a file (${labelOf(file)}): ${why}` })
const one = async (sent: MessageAttachment, signal: AbortSignal | undefined) => {
const label = labelOf(sent)
let file = sent, failure: unknown, stale = false
for (let attempt = 0; attempt < 3 && deadline - now() > 0 && !signal?.aborted; attempt++) {
try {
if (options.relink && (stale || file.unlinked || (file.expiresAt !== undefined && file.expiresAt - now() < RELINK_MARGIN_MS))) {
try { file = await options.relink(file) } catch (error) {
// Refused (404) or forbidden (403): its reader may no longer read the message, or the file is gone.
const status = error && typeof error === 'object' && 'status' in error ? error.status : null
throw new Refused(status === 404 || status === 403 ? 'no longer available' : 'could not get a fresh link')
}
stale = false
}
const path = await download(file, base, signedBase, options.fetch ?? fetch, Math.min(downloadMs(file.size), deadline - now()),
options.idleMs ?? IDLE_MS, signal, options.statfs ?? fsStatfs)
return { line: `- ${path} (${label})`, path }
} catch (error) {
failure = error
if (error instanceof Refused) break
if (error instanceof Expired) {
if (!options.relink) { failure = file.unlinked ? new Error('could not get a link just now') : error; break }
stale = true
}
}
}
return failed(file, signal?.aborted ? 'stopped' : failure instanceof Error ? failure.message : 'out of time')
}
const lines = await Promise.all(files.map(async file => {
const inFlight = options.inFlight
if (!inFlight) return one(file, options.signal)
// By the file, not its link: each read signs a link of its own.
const key = `${file.size} ${file.id || file.url.split('?')[0]}`
let shared = inFlight.get(key)
const joined = !!shared
if (!shared) {
const stop = new AbortController()
const started: SharedDownload = { wanting: 0, stop,
result: one(file, stop.signal).finally(() => { if (inFlight.get(key) === started) inFlight.delete(key) }) }
inFlight.set(key, shared = started)
}
const download = shared
// This delivery wants it until it is in or the delivery leaves; the last one to leave stops it, and its part file goes.
download.wanting++
let left = false
const leave = () => { if (left) return; left = true; if (--download.wanting === 0) download.stop.abort() }
if (options.signal?.aborted) leave()
else options.signal?.addEventListener('abort', leave, { once: true })
const result = await download.result
options.signal?.removeEventListener('abort', leave)
if (!joined || result.path === null) return result
// Downloaded once, but each delivery's own file: copied as soon as the download is in, before either is delivered.
try {
const path = await copyPasteDropFile(result.path, diskName(file.name))
return { line: `- ${path} (${labelOf(file)})`, path }
} catch (error) { return failed(file, isNoRoom(error) ? NO_ROOM : 'could not copy') }
}))
const paths = lines.flatMap(l => l.path === null ? [] : [l.path])
return { text: `Attached files:\n${lines.map(l => l.line).join('\n')}`, complete: paths.length === lines.length, paths }
}