The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
import { open, stat, type FileHandle } from 'node:fs/promises'
import { createReadStream } from 'node:fs'
import { createInterface } from 'node:readline'
import { verifyTranscriptHandle, type TranscriptFileIdentity } from './transcriptBoundary.js'
const CHUNK_BYTES = 64 * 1024
/** A record longer than this is passed over unread by the walks below. No engine writes one — Codex's
* compaction snapshots, the largest, are tens of MB — and one over V8's string limit (~512 MiB) could
* not be decoded anyway. Holding one while looking for its end is the only thing either walk keeps. */
export const MAX_RECORD_BYTES = 256 * 1024 * 1024
/** `line.trim()` is empty — decided without decoding a record that has a visible ASCII character in it,
* which is every JSONL record (they open with `{`). */
function isBlank(bytes: Buffer): boolean {
for (const byte of bytes) if (byte > 0x20 && byte < 0x7f) return false
return !bytes.toString('utf8').trim()
}
async function readFully(handle: FileHandle, chunk: Buffer, position: number): Promise<boolean> {
let read = 0
while (read < chunk.length) {
const { bytesRead } = await handle.read(chunk, read, chunk.length - read, position + read)
if (!bytesRead) return false
read += bytesRead
}
return true
}
/**
* Walk a transcript's records from byte `end` back to byte 0, newest first, holding one at a time.
*
* `visit` gets each non-blank record's bytes — valid only during the call — and the offset it starts
* at; returning true stops the walk. LF, CR and CRLF each end a record, readline's rule. A record over
* `maxRecordBytes` is passed over unvisited, and only told to `passed` (its offset), if given; returning
* true from that stops the walk too. Resolves false when the file shrank under the walk: what was
* visited may then mix two versions of it.
*/
export async function scanRecordsBackward(
filePath: string,
end: number,
visit: (record: Buffer, offset: number) => boolean | void,
maxRecordBytes = MAX_RECORD_BYTES,
passed?: (offset: number) => boolean | void,
expectedFile?: TranscriptFileIdentity,
): Promise<boolean> {
const handle = await open(filePath, 'r')
try {
if (expectedFile) await verifyTranscriptHandle(handle, expectedFile)
let position = end
// Pieces of the record being assembled, newest first; only ever one record's worth.
let fragments: Buffer[] = []
let fragmentBytes = 0
let oversized = false
const take = (head: Buffer, offset: number): boolean => {
const skip = oversized || head.length + fragmentBytes > maxRecordBytes
const bytes = skip || !fragments.length ? head : Buffer.concat([head, ...fragments.reverse()])
fragments = []
fragmentBytes = 0
oversized = false
if (skip) return passed?.(offset) === true
if (isBlank(bytes)) return false
return visit(bytes, offset) === true
}
while (position > 0) {
const length = Math.min(position, CHUNK_BYTES)
position -= length
const chunk = Buffer.allocUnsafe(length)
if (!await readFully(handle, chunk, position)) return false
let segmentEnd = length
// Each native search advances independently: CR-free files must not rescan the
// remaining chunk for CR once per LF. No per-byte JavaScript loop is needed.
let lf = chunk.lastIndexOf(10), cr = chunk.lastIndexOf(13)
while (lf >= 0 || cr >= 0) {
const i = Math.max(lf, cr)
// Match readline's LF, CRLF and bare-CR handling. The empty CRLF half is blank and skipped.
if (take(chunk.subarray(i + 1, segmentEnd), position + i + 1)) return true
segmentEnd = i
if (lf === i) lf = i > 0 ? chunk.lastIndexOf(10, i - 1) : -1
if (cr === i) cr = i > 0 ? chunk.lastIndexOf(13, i - 1) : -1
}
if (segmentEnd && !oversized) {
fragmentBytes += segmentEnd
if (fragmentBytes > maxRecordBytes) { oversized = true; fragments = [] }
else fragments.push(chunk.subarray(0, segmentEnd))
}
}
if (fragments.length || oversized) take(Buffer.alloc(0), 0)
return true
} finally { await handle.close() }
}
/**
* Feed every non-blank record in [start, end) to `onRecord` with the offset it starts at, oldest first,
* holding one at a time — the forward half of `scanRecordsBackward`, with the same record rules.
* Returning true from `onRecord` stops after that record.
*
* A final record with no line ending yet is fed only when `complete` accepts it: an engine caught
* mid-write leaves half a record there, and it belongs to whoever reads the file next (`partial`). `next`
* is the offset just past the last record fed — exactly where a tail picks up. Null when the file shrank.
*/
export async function streamRecords(
filePath: string,
start: number,
end: number,
onRecord: (line: string, offset: number) => boolean | void,
complete: (line: string) => boolean,
maxRecordBytes = MAX_RECORD_BYTES,
expectedFile?: TranscriptFileIdentity,
): Promise<{ next: number; records: number; partial: boolean } | null> {
const handle = await open(filePath, 'r')
try {
if (expectedFile) await verifyTranscriptHandle(handle, expectedFile)
let position = start
let next = start
let records = 0
let partial = false
let pending: Buffer[] = []
let pendingBytes = 0
let oversized = false
const emit = (bytes: Buffer): boolean => {
if (isBlank(bytes)) return false
records++
return onRecord(bytes.toString('utf8'), next) === true
}
while (position < end) {
const length = Math.min(end - position, CHUNK_BYTES)
const chunk = Buffer.allocUnsafe(length)
if (!await readFully(handle, chunk, position)) return null
let segmentStart = 0
let lf = chunk.indexOf(10), cr = chunk.indexOf(13)
while (lf >= 0 || cr >= 0) {
const i = lf < 0 ? cr : cr < 0 ? lf : Math.min(lf, cr)
const piece = chunk.subarray(segmentStart, i)
const stop = !oversized && pendingBytes + piece.length <= maxRecordBytes
&& emit(pending.length ? Buffer.concat([...pending, piece]) : piece)
pending = []
pendingBytes = 0
oversized = false
segmentStart = i + 1
next = position + i + 1
if (stop) return { next, records, partial }
if (lf === i) lf = chunk.indexOf(10, i + 1)
if (cr === i) cr = chunk.indexOf(13, i + 1)
}
if (segmentStart < length && !oversized) {
pendingBytes += length - segmentStart
if (pendingBytes > maxRecordBytes) { oversized = true; pending = [] }
else pending.push(chunk.subarray(segmentStart))
}
position += length
}
if (pending.length) {
const bytes = Buffer.concat(pending)
const line = bytes.toString('utf8')
if (!isBlank(bytes)) {
if (complete(line)) {
records++
onRecord(line, next)
next = end
} else partial = true
}
}
return { next, records, partial }
} finally { await handle.close() }
}
/** Read selected JSONL records backward, then return them in chronological order.
* `stop` includes the boundary record; `skip` discards an irrelevant record immediately.
* Only complete records are decoded, so UTF-8 may cross any read boundary. The file length is
* captured at open: later appends belong to the next read. A missing boundary reads back to BOF. */
export async function tailFileUntil(filePath: string, select: (line: string) => 'keep' | 'skip' | 'stop'): Promise<string[]> {
try {
const { size } = await stat(filePath)
const lines: string[] = []
const whole = await scanRecordsBackward(filePath, size, (record) => {
const line = record.toString('utf8')
const action = select(line)
if (action !== 'skip') lines.push(line)
return action === 'stop'
})
// Truncated during the read; don't recap a mixed snapshot.
return whole ? lines.reverse() : []
} catch {
return []
}
}
/** The most of a transcript read at once by a reader that has no pages of its own: its newest records
* up to this many bytes. The October 3 crash was an 803 MB transcript read whole: past what the heap
* holds, one conversation's read takes every agent's daemon down with it. Claude Code and Codex read
* pages instead (lib/transcriptPages.ts); this is the floor under every other engine until they do. */
export const WHOLE_READ_CAP_BYTES = 64 * 1024 * 1024
/**
* A transcript's newest records, up to `maxBytes` of them, in order: what `tailFile(path, Infinity)`
* returns for a file under the cap, with `truncated` saying older records were left unread. A record
* that alone is bigger than what is left of the cap ends the read there. Empty for a file that cannot
* be read.
*/
export async function tailFileCapped(filePath: string, maxBytes = WHOLE_READ_CAP_BYTES): Promise<{ lines: string[]; truncated: boolean }> {
try {
const { size } = await stat(filePath)
const lines: string[] = []
let bytes = 0
let truncated = false
await scanRecordsBackward(filePath, size, (record) => {
if (bytes + record.length > maxBytes) { truncated = true; return true }
bytes += record.length
lines.push(record.toString('utf8'))
})
return { lines: lines.reverse(), truncated }
} catch {
return { lines: [], truncated: false }
}
}
/** Return the last `n` non-empty raw lines of a specific transcript file. Prefer this over
* `tailLines` when the caller already holds a trusted, registered `transcriptPath` — it takes no
* request-controlled id, so there is no path to traverse. */
export async function tailFile(filePath: string, n = 200): Promise<string[]> {
try {
if (n === Infinity) {
// A long-running session can exceed V8's maximum single-string length.
// Keep the full-history contract, without decoding the entire file at once.
const lines: string[] = []
const input = createReadStream(filePath, { encoding: 'utf8' })
const reader = createInterface({ input, crlfDelay: Infinity })
try {
for await (const line of reader) if (line.trim()) lines.push(line)
} finally { reader.close(); input.destroy() }
return lines
}
if (!Number.isFinite(n)) return []
n = Math.floor(n)
if (n <= 0) return []
const handle = await open(filePath, 'r')
try {
let position = (await handle.stat()).size
const chunks: Buffer[] = []
let newlines = 0
while (position > 0) {
const size = Math.min(position, 64 * 1024)
position -= size
const chunk = Buffer.allocUnsafe(size)
const { bytesRead } = await handle.read(chunk, 0, size, position)
const read = chunk.subarray(0, bytesRead)
chunks.unshift(read)
for (const byte of read) if (byte === 10) newlines++
if (newlines > n || position === 0) {
let text = Buffer.concat(chunks).toString('utf8')
if (position > 0) text = text.slice(text.indexOf('\n') + 1)
const lines = text.split('\n').filter(line => line.trim())
if (lines.length >= n || position === 0) return lines.slice(-n)
}
}
return []
} finally { await handle.close() }
} catch {
return []
}
}