Multiplayer agent harness for work.
Count inline documents in the context budget and stub them when they no longer fit
Joshua France71de5363b72c
diff --git a/src/core/attachments.ts b/src/core/attachments.tsindex 580a8952d..52fc63f57 100644--- a/src/core/attachments.ts+++ b/src/core/attachments.ts@@ -18,6 +18,7 @@ import {} from "../persistence/blob-transfer.ts";import { fileArtifactId, type FileArtifactStore, type FileDirection } from "../files/file-artifact-store.ts";import { swallowAs } from "../util/errors.ts";+import { fileContentRequest } from "./document-kinds.ts";import { hashId } from "../util/crypto.ts";import type { SecurityScreenVerdict } from "../security/security-posture.ts";import { downscaleVisionImage } from "./image-downscale.ts";@@ -212,7 +213,7 @@ export function inboundManifest(metas: AttachmentMeta[], inboxDir = INBOX_DIR, u(m) =>`- ${inboxDir}/${m.name} (${m.mimetype}, ${m.sizeBytes} bytes)${m.author ? ` — shared by ${m.author}` : ""}${unstaged?.has(m.name)- ? ` — saved as file ${m.artifactId}, not yet on a computer (no default sandbox); send it directly with {file_id: "${m.artifactId}"}; if you need it on disk, choose a sandbox_id for execute and download GET $AGENT_API_URL/v1/files/${m.artifactId}/content (header x-agent-capability: $AGENT_API_TOKEN) to a path you choose`+ ? ` — saved as file ${m.artifactId}, not yet on a computer (no default sandbox); send it directly with {file_id: "${m.artifactId}"}; if you need it on disk, choose a sandbox_id for execute and download ${fileContentRequest(m.artifactId!)} to a path you choose`: ""}`,)diff --git a/src/core/document-inputs.ts b/src/core/document-inputs.tsindex 81613c5d5..d25f74fc6 100644--- a/src/core/document-inputs.ts+++ b/src/core/document-inputs.ts@@ -5,6 +5,16 @@ import type { AttachmentMeta, SessionEntry } from "../types.ts";import type { FileArtifact, FileArtifactStore } from "../files/file-artifact-store.ts";import { forModelContext } from "../harness/context-compaction.ts";import { collectBytes } from "../util/bytes.ts";+import {+ MAX_DOCUMENT_BYTES,+ MAX_DOCUMENT_TEXT_CHARS,+ OFFICE_FALLBACK_EXTENSIONS,+ documentExtension,+ documentStub,+ estimateDocumentTokens,+ isDocumentAttachment,+ isTextDocument,+} from "./document-kinds.ts";export interface DocumentInput {name: string;@@ -13,56 +23,6 @@ export interface DocumentInput {artifactId?: string;}-export const MAX_DOCUMENT_BYTES = 8_000_000;-const MAX_DOCUMENT_TEXT_CHARS = 200_000;-const TEXT_EXTENSIONS = new Set(- "asm bat c cc conf cpp css cxx def dic eml h hh htm html ics ifb in js json ksh list log markdown md mht mhtml mime mjs nws pl py rst s sql srt text txt vcf vtt xml ts tsx jsx sh bash zsh yml yaml toml rs go java rb php swift kt scala lua r jl perl tex cs graphql ndjson json5 dockerfile".split(- " ",- ),-);-const DOCUMENT_EXTENSIONS = new Set(- "pdf xla xlb xlc xlm xls xlsx xlt xlw csv tsv iif doc docx dot odt rtf pages pot ppa pps ppt pptx pwz wiz key ods odp".split(- " ",- ),-);-const OFFICE_FALLBACK_EXTENSIONS = new Set("pdf docx pptx xlsx odt odp ods rtf".split(" "));-const extension = (name: string): string => name.split(".").at(-1)!.toLowerCase();--const MIME_EXTENSIONS: Record<string, string> = {- "application/pdf": "pdf",- "application/msword": "doc",- "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",- "application/vnd.ms-excel": "xls",- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",- "application/vnd.ms-powerpoint": "ppt",- "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",- "application/vnd.oasis.opendocument.text": "odt",- "application/vnd.oasis.opendocument.spreadsheet": "ods",- "application/vnd.oasis.opendocument.presentation": "odp",- "application/rtf": "rtf",- "text/rtf": "rtf",- "application/json": "json",- "application/xml": "xml",- "application/javascript": "js",-};--export function documentExtension(file: { name: string; mimeType?: string; mimetype?: string }): string {- const ext = extension(file.name);- if (TEXT_EXTENSIONS.has(ext) || DOCUMENT_EXTENSIONS.has(ext)) return ext;- const mime = (file.mimeType ?? file.mimetype ?? "").split(";")[0]!.trim().toLowerCase();- return MIME_EXTENSIONS[mime] ?? (mime.startsWith("text/") ? "txt" : ext);-}--export function isDocumentAttachment(file: { name: string; mimetype?: string }): boolean {- const ext = documentExtension(file);- return TEXT_EXTENSIONS.has(ext) || DOCUMENT_EXTENSIONS.has(ext);-}--export function isTextDocument(file: Pick<DocumentInput, "name" | "mimeType">): boolean {- const ext = documentExtension(file);- return TEXT_EXTENSIONS.has(ext) || ["csv", "tsv", "iif"].includes(ext);-}-export function historicalDocumentMetas(history: readonly SessionEntry[]): AttachmentMeta[] {return forModelContext([...history]).flatMap((entry) => {if (entry.type !== "user") return [];@@ -83,7 +43,8 @@ export async function loadDocumentInputs(signal?: AbortSignal,maxDocuments = 10,existingDocuments: readonly DocumentInput[] = [],-): Promise<{ documents: DocumentInput[]; notices: string[] }> {+ maxTokens = Number.POSITIVE_INFINITY,+): Promise<{ documents: DocumentInput[]; notices: string[]; tokens: number }> {const documents: DocumentInput[] = [];const notices: string[] = [];const seen = new Set<string>();@@ -97,6 +58,7 @@ export async function loadDocumentInputs(),);let remaining = maxBytes;+ let usedTokens = 0;for (const meta of [...metas].reverse()) {signal?.throwIfAborted();try {@@ -106,10 +68,9 @@ export async function loadDocumentInputs(if (!artifact || !(await authorize(artifact)) || !artifact.sha256 || !isDocumentAttachment(artifact)) continue;const contentKey = JSON.stringify([artifact.sha256, artifact.name, artifact.mimetype]);if (loadedContent.has(contentKey)) continue;- if (artifact.sizeBytes > remaining || documents.length >= maxDocuments) {- notices.push(- `${artifact.name}: document omitted from model input because it exceeds the attachment count or byte budget.`,- );+ const tokens = estimateDocumentTokens(artifact);+ if (artifact.sizeBytes > remaining || documents.length >= maxDocuments || usedTokens + tokens > maxTokens) {+ notices.push(documentStub({ name: artifact.name, artifactId: artifact.id }));continue;}const opened = await files.open(meta.artifactId);@@ -137,6 +98,7 @@ export async function loadDocumentInputs(continue;}remaining -= bytes.sizeBytes;+ usedTokens += tokens;loadedContent.add(contentKey);documents.unshift({name: artifact.name,@@ -149,7 +111,7 @@ export async function loadDocumentInputs(notices.push(`${meta.name}: document content is unavailable.`);}}- return { documents, notices };+ return { documents, notices, tokens: usedTokens };}export interface ExtractedDocumentText {diff --git a/src/core/document-kinds.ts b/src/core/document-kinds.tsnew file mode 100644index 000000000..4ede280e0--- /dev/null+++ b/src/core/document-kinds.ts@@ -0,0 +1,95 @@+import type { AttachmentMeta } from "../types.ts";++export const MAX_DOCUMENT_BYTES = 8_000_000;+export const MAX_DOCUMENT_TEXT_CHARS = 200_000;+const TEXT_EXTENSIONS = new Set(+ "asm bat c cc conf cpp css cxx def dic eml h hh htm html ics ifb in js json ksh list log markdown md mht mhtml mime mjs nws pl py rst s sql srt text txt vcf vtt xml ts tsx jsx sh bash zsh yml yaml toml rs go java rb php swift kt scala lua r jl perl tex cs graphql ndjson json5 dockerfile".split(+ " ",+ ),+);+const DOCUMENT_EXTENSIONS = new Set(+ "pdf xla xlb xlc xlm xls xlsx xlt xlw csv tsv iif doc docx dot odt rtf pages pot ppa pps ppt pptx pwz wiz key ods odp".split(+ " ",+ ),+);+export const OFFICE_FALLBACK_EXTENSIONS = new Set("pdf docx pptx xlsx odt odp ods rtf".split(" "));+const extension = (name: string): string => name.split(".").at(-1)!.toLowerCase();++const MIME_EXTENSIONS: Record<string, string> = {+ "application/pdf": "pdf",+ "application/msword": "doc",+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",+ "application/vnd.ms-excel": "xls",+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",+ "application/vnd.ms-powerpoint": "ppt",+ "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",+ "application/vnd.oasis.opendocument.text": "odt",+ "application/vnd.oasis.opendocument.spreadsheet": "ods",+ "application/vnd.oasis.opendocument.presentation": "odp",+ "application/rtf": "rtf",+ "text/rtf": "rtf",+ "application/json": "json",+ "application/xml": "xml",+ "application/javascript": "js",+};++export function documentExtension(file: { name: string; mimeType?: string; mimetype?: string }): string {+ const ext = extension(file.name);+ if (TEXT_EXTENSIONS.has(ext) || DOCUMENT_EXTENSIONS.has(ext)) return ext;+ const mime = (file.mimeType ?? file.mimetype ?? "").split(";")[0]!.trim().toLowerCase();+ return MIME_EXTENSIONS[mime] ?? (mime.startsWith("text/") ? "txt" : ext);+}++export function isDocumentAttachment(file: { name: string; mimetype?: string }): boolean {+ const ext = documentExtension(file);+ return TEXT_EXTENSIONS.has(ext) || DOCUMENT_EXTENSIONS.has(ext);+}++export function isTextDocument(file: { name: string; mimeType?: string; mimetype?: string }): boolean {+ const ext = documentExtension(file);+ return TEXT_EXTENSIONS.has(ext) || ["csv", "tsv", "iif"].includes(ext);+}++// Conservative per-byte estimate of what an attachment costs once expanded into a model request:+// native PDFs are billed per page (text plus a page image), which tracks file size far better than+// the few words a session entry stores for the attachment itself.+const DOCUMENT_BYTES_PER_TOKEN = 4;++export function estimateDocumentTokens(file: {+ name: string;+ mimeType?: string;+ mimetype?: string;+ sizeBytes: number;+}): number {+ const bytes = Number.isFinite(file.sizeBytes) && file.sizeBytes > 0 ? file.sizeBytes : 0;+ const expanded = isTextDocument(file) ? Math.min(bytes, MAX_DOCUMENT_TEXT_CHARS) : bytes;+ return Math.ceil(expanded / DOCUMENT_BYTES_PER_TOKEN);+}++export function replayableDocumentMetas(payload: unknown): AttachmentMeta[] {+ const attachments = (payload as { attachments?: unknown } | null)?.attachments;+ if (!Array.isArray(attachments)) return [];+ return (attachments as AttachmentMeta[]).filter(+ (file) =>+ file?.direction === "in" &&+ typeof file.name === "string" &&+ typeof file.artifactId === "string" &&+ isDocumentAttachment(file),+ );+}++export function documentAttachmentTokens(payload: unknown): number {+ return replayableDocumentMetas(payload).reduce((sum, file) => sum + estimateDocumentTokens(file), 0);+}++/** How the agent fetches a saved file's bytes through its self-API. */+export function fileContentRequest(artifactId: string): string {+ return `GET $AGENT_API_URL/v1/files/${artifactId}/content (header x-agent-capability: $AGENT_API_TOKEN)`;+}++export function documentStub(file: { name: string; artifactId?: string }): string {+ const where = file.artifactId+ ? `It is still saved as file ${file.artifactId}: read it with ${fileContentRequest(file.artifactId)} and extract only the parts you need.`+ : "It is still saved in this conversation's files.";+ return `${JSON.stringify(file.name)} is not included inline, to keep this conversation within the model's context. ${where}`;+}diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex e9d4db3e2..a7df17e10 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -12,13 +12,12 @@ import { UNTRACKED_PROVENANCE } from "../memory/facts.ts";import { memoryRecallQuery } from "../memory/semantic-recall.ts";import { requiresDelegation, delegatedAuthorizationOrigin } from "../sessions/session-syscalls.ts";import {- MAX_DOCUMENT_BYTES,documentText,type ExtractedDocumentText,historicalDocumentMetas,- isTextDocument,loadDocumentInputs,} from "./document-inputs.ts";+import { MAX_DOCUMENT_BYTES, estimateDocumentTokens, isDocumentAttachment, isTextDocument } from "./document-kinds.ts";import { recoveredRuntime } from "../harness/runtime-recovery.ts";import { createCanWriteScope, rosterAdmits, withLiveTurnMembership } from "../resolution/scope-membership.ts";import { goalView } from "../runs/turn-stream.ts";@@ -292,7 +291,8 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {}const securitySteersInFlight = new Set<string>();- const { compactContextIfNeeded, compactRecent, scheduleBackgroundCompaction } = createCompaction(deps);+ const { compactContextIfNeeded, compactRecent, scheduleBackgroundCompaction, documentTokenBudget } =+ createCompaction(deps);async function approvalSummary(scopeId: ScopeId,@@ -2835,6 +2835,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {resumePlan = resumeStrategy(visibleHistory, partial);}const resume = resumePlan && resumePlan.kind !== "restart" ? partial : null;+ const documentBudget = documentTokenBudget(scopeId, input.model);const history = await withManagedRosterVersion(() =>compactContextIfNeeded({cancel: turnAbort.signal,@@ -2845,19 +2846,28 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {orgScopeId: resolution.orgScopeId,actorId: actor.id,...(input.model ? { model: input.model } : {}),+ reservedTokens: strictReadOnly+ ? 0+ : inbound.metas.reduce(+ (sum, meta) => sum + (isDocumentAttachment(meta) ? estimateDocumentTokens(meta) : 0),+ 0,+ ),}),);contextRecovered =history !== visibleHistory &&history.some((entry) => isObj(entry.payload) && entry.payload.mode === "recent");const documentInputs = strictReadOnly- ? { documents: [], notices: [] }+ ? { documents: [], notices: [], tokens: 0 }: await loadDocumentInputs(deps.files,[...historicalDocumentMetas(history), ...inbound.metas],mayReadArtifact,undefined,turnAbort.signal,+ undefined,+ undefined,+ documentBudget,);const screenDocuments = async (documentInputs: Awaited<ReturnType<typeof loadDocumentInputs>>,@@ -2920,6 +2930,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {MAX_DOCUMENT_BYTES -documentInputs.documents.reduce((sum, document) => sum + Buffer.byteLength(document.dataBase64, "base64"), 0);let remainingDocumentCount = 10 - documentInputs.documents.length;+ let remainingDocumentTokens = documentBudget - documentInputs.tokens;const principalDelivered = await recentPrincipalDeliveryNote(deps.deliveries,session.threadRef,@@ -3351,6 +3362,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {turnAbort.signal,remainingDocumentCount,initialDocuments,+ remainingDocumentTokens,);let steeredUnscreened = false;if (enforceScreen) steeredUnscreened = await screenDocuments(steeredDocuments, text);@@ -3360,6 +3372,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {0,);remainingDocumentCount -= steeredDocuments.documents.length;+ remainingDocumentTokens -= steeredDocuments.tokens;documentInputs.documents.push(...steeredDocuments.documents);const issues = inboundIssueList({...received,diff --git a/src/core/orchestrator/compaction.ts b/src/core/orchestrator/compaction.tsindex 516fcc24e..67b957c13 100644--- a/src/core/orchestrator/compaction.ts+++ b/src/core/orchestrator/compaction.ts@@ -17,7 +17,10 @@ import {overBudgetFraction,planCompaction,recentEntryCountWithinBudget,+ summarizedDocuments,+ withSummarizedDocuments,} from "../../harness/context-compaction.ts";+import type { SummarizedDocument } from "../../sessions/session-store.ts";import { estimateCostUsd } from "../../ratelimit/budget.ts";import { errMessage } from "../../util/errors.ts";import { createKeyedQueue } from "../../util/async.ts";@@ -25,12 +28,14 @@ import type { OrchestratorDeps } from "./types.ts";const MAX_CONTEXT_TOKENS = 120_000;const KEEP_RECENT_TOKEN_FRACTION = 0.6;+const DOCUMENT_TOKEN_FRACTION = 0.5;interface Summarized {text: string;summaryLabel: ScopeId;throughSeq: number;mode?: "recent";+ documents?: SummarizedDocument[];}export interface CompactionContext {@@ -43,6 +48,8 @@ export interface CompactionContext {actorId: string;model?: string;cancel?: AbortSignal;+ /** Tokens this turn's inline documents will add on top of the history. */+ reservedTokens?: number;}): Promise<SessionEntry[]>;compactRecent(input: {session: Session;@@ -53,6 +60,7 @@ export interface CompactionContext {actorId: string;model?: string;cancel?: AbortSignal;+ reservedTokens?: number;}): Promise<SessionEntry[]>;scheduleBackgroundCompaction(input: {sessionId: string;@@ -61,11 +69,20 @@ export interface CompactionContext {actorId: string;model?: string;}): void;+ /** Most of the context budget that inline documents may take in one model request. */+ documentTokenBudget(scopeId: string, model?: string): number;}export function createCompaction(deps: OrchestratorDeps): CompactionContext {const tokenBudgetFor = (scopeLabel?: string, model?: string): number =>deps.maxContextTokens ?? deps.harness.models.contextTokenBudget?.(scopeLabel, model) ?? MAX_CONTEXT_TOKENS;+ const documentTokenBudget = (scopeId: string, model?: string): number =>+ Math.floor(tokenBudgetFor(scopeId, model) * DOCUMENT_TOKEN_FRACTION);+ const historyBudgetFor = (input: { scopeId: string; model?: string; reservedTokens?: number }): number => {+ const total = tokenBudgetFor(input.scopeId, input.model);+ const reserved = Math.min(Math.max(0, input.reservedTokens ?? 0), documentTokenBudget(input.scopeId, input.model));+ return Math.max(1, total - reserved);+ };const boundRecent = (entries: SessionEntry[], maxContextTokens: number): SessionEntry[] => {const summary = entries.find((e) => contextSummaryPayload(e));@@ -91,12 +108,13 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {actorId: string;model?: string;cancel?: AbortSignal;+ reservedTokens?: number;},recoverOnFailure = false,): Promise<Summarized | null> {if (isManagedGroupScope(input.scopeId)) return null;if (!deps.harness.models.compactHistory) return null;- const maxContextTokens = tokenBudgetFor(input.scopeId, input.model);+ const maxContextTokens = historyBudgetFor(input);const reuseBudgetTokens = Math.floor(maxContextTokens * keepRecentTokenFraction);const plan = planCompaction(input.visibleHistory, maxContextTokens, keepRecentTokenFraction, reuseBudgetTokens);if (!plan || !plan.toSummarize.length) return null;@@ -135,6 +153,7 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {text,summaryLabel,throughSeq: compactionThroughSeq(plan.toSummarize),+ documents: summarizedDocuments(plan.toSummarize),};}@@ -143,11 +162,11 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {lease: Lease;summarized: Summarized;}): Promise<SessionEntry> {- const { text, summaryLabel, throughSeq, mode } = input.summarized;+ const { text, summaryLabel, throughSeq, mode, documents = [] } = input.summarized;const summary = await deps.sessions.append(input.lease, {type: "system",payload: {- ...createContextSummaryPayload(throughSeq, text),+ ...createContextSummaryPayload(throughSeq, withSummarizedDocuments(text, documents), documents),...(mode ? { mode } : {}),},scopeLabel: summaryLabel,@@ -165,6 +184,7 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {actorId: string;model?: string;cancel?: AbortSignal;+ reservedTokens?: number;}): Promise<SessionEntry[] | null> {const summarized = await summarizeForCompaction(input, true);if (!summarized) return null;@@ -172,7 +192,7 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {const recent = input.visibleHistory.filter((entry) => entry.seq > summarized.throughSeq && !contextSummaryPayload(entry),);- return boundRecent([summary, ...recent], tokenBudgetFor(input.scopeId, input.model));+ return boundRecent([summary, ...recent], historyBudgetFor(input));}async function compactContextIfNeeded(input: {@@ -184,8 +204,9 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {actorId: string;model?: string;cancel?: AbortSignal;+ reservedTokens?: number;}): Promise<SessionEntry[]> {- const maxContextTokens = tokenBudgetFor(input.scopeId, input.model);+ const maxContextTokens = historyBudgetFor(input);if (!overBudgetFraction(input.visibleHistory, maxContextTokens, COMPACT_HARD_FRACTION)) {return input.visibleHistory;}@@ -200,7 +221,7 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {async function compactRecent(input: Parameters<CompactionContext["compactRecent"]>[0]): Promise<SessionEntry[]> {input.cancel?.throwIfAborted();- const budget = Math.floor(tokenBudgetFor(input.scopeId, input.model) * keepRecentTokenFraction);+ const budget = Math.floor(historyBudgetFor(input) * keepRecentTokenFraction);const prior = input.visibleHistory.findLast((entry) => contextSummaryPayload(entry));const rest = input.visibleHistory.filter((entry) => !contextSummaryPayload(entry));const notice =@@ -253,6 +274,7 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {summaryLabel: (includedPrior?.scopeLabel ?? input.scopeId) as ScopeId,throughSeq,mode: "recent",+ documents: summarizedDocuments([...(prior ? [prior] : []), ...rest.slice(0, start)]),},});return [summary, ...recent];@@ -326,5 +348,5 @@ export function createCompaction(deps: OrchestratorDeps): CompactionContext {});}- return { compactContextIfNeeded, compactRecent, scheduleBackgroundCompaction };+ return { compactContextIfNeeded, compactRecent, scheduleBackgroundCompaction, documentTokenBudget };}diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.tsindex 76f2162b4..e77180f52 100644--- a/src/harness/agent-tools.ts+++ b/src/harness/agent-tools.ts@@ -72,6 +72,8 @@ function describePublishAudience(a: PublishAudienceDescriptor | undefined): striexport interface ToolContextRef {documents?: DocumentInput[];+ /** Set once a provider rejects a request carrying `documents`; later requests send stubs instead. */+ documentsStubbed?: boolean;runtimeHandoff?: HarnessHandoff;runtimeRunId?: string;runtimeActorId?: string;diff --git a/src/harness/context-compaction.ts b/src/harness/context-compaction.tsindex ab0064af8..8ff7ebd74 100644--- a/src/harness/context-compaction.ts+++ b/src/harness/context-compaction.ts@@ -1,9 +1,10 @@import type { RunStop } from "../runs/run-store.ts";import type { SessionEntry } from "../types.ts";-import { contextSummaryPayload } from "../sessions/session-store.ts";+import { contextSummaryPayload, type SummarizedDocument } from "../sessions/session-store.ts";import { headSlice, tailSlice } from "../util/text.ts";import { countTokens } from "../util/tokens.ts";import { isObj } from "../util/objects.ts";+import { documentAttachmentTokens, fileContentRequest, replayableDocumentMetas } from "../core/document-kinds.ts";const MAX_COMPACT_ENTRY_CHARS = 16_000;const COMPACT_ENTRY_TAIL_CHARS = 2_000;@@ -78,6 +79,10 @@ const entryTokenCache = new Map<string, number>();const ENTRY_TOKEN_CACHE_MAX = 50_000;export function estimateEntryTokens(entry: SessionEntry): number {+ return estimateEntryTextTokens(entry) + (entry.type === "user" ? documentAttachmentTokens(entry.payload) : 0);+}++function estimateEntryTextTokens(entry: SessionEntry): number {const payload = entry.payload as { text?: string; environment?: string; thinking?: string } | null;const prose = entry.type === "thinking" ? [payload?.thinking] : [payload?.text, payload?.environment];const text =@@ -218,6 +223,39 @@ export function planCompaction(return { toSummarize, kept: afterSummary.slice(overflowCount) };}+const SUMMARIZED_DOCUMENTS_HEADER = `[Files shared in the summarized turns are no longer included inline. Each is still saved; read one with ${fileContentRequest("<file id>")} when you need its contents.]`;+const MAX_SUMMARIZED_DOCUMENTS = 20;+const MAX_SUMMARIZED_DOCUMENT_NAME_CHARS = 200;++/** Documents attached across `entries` (carrying forward any listed by a prior summary), newest last. */+export function summarizedDocuments(entries: readonly SessionEntry[]): SummarizedDocument[] {+ const byId = new Map<string, SummarizedDocument>();+ for (const entry of entries) {+ const documents =+ contextSummaryPayload(entry)?.documents ??+ (entry.type === "user"+ ? replayableDocumentMetas(entry.payload).map((meta) => ({ name: meta.name, artifactId: meta.artifactId! }))+ : []);+ for (const document of documents) {+ byId.delete(document.artifactId);+ byId.set(document.artifactId, {+ name: headSlice(document.name, MAX_SUMMARIZED_DOCUMENT_NAME_CHARS),+ artifactId: document.artifactId,+ });+ }+ }+ return [...byId.values()].slice(-MAX_SUMMARIZED_DOCUMENTS);+}++/** Replaces the summary's trailing file list so documents dropped from model input stay discoverable. */+export function withSummarizedDocuments(text: string, documents: readonly SummarizedDocument[]): string {+ const at = text.indexOf(`\n\n${SUMMARIZED_DOCUMENTS_HEADER}`);+ const base = at >= 0 ? text.slice(0, at) : text;+ if (!documents.length) return base;+ const list = documents.map((document) => `- ${JSON.stringify(document.name)} (file ${document.artifactId})`);+ return `${base}\n\n${SUMMARIZED_DOCUMENTS_HEADER}\n${list.join("\n")}`;+}+export function compactionThroughSeq(entries: SessionEntry[]): number {return entries.reduce((max, entry) => {const summary = contextSummaryPayload(entry);diff --git a/src/harness/document-inputs.ts b/src/harness/document-inputs.tsindex b69a26b79..51828edb7 100644--- a/src/harness/document-inputs.ts+++ b/src/harness/document-inputs.ts@@ -1,15 +1,14 @@import { resolveGatewayModel } from "../model/gateway-models.ts";import { countTokens } from "../util/tokens.ts";import {- documentExtension,fitDocumentText,type DocumentTextBudget,documentFallbackText,documentText,- isTextDocument,nativeDocumentIsReadable,type DocumentInput,} from "../core/document-inputs.ts";+import { documentExtension, isTextDocument } from "../core/document-kinds.ts";export interface DocumentModel {api?: string;@@ -130,11 +129,16 @@ export async function withDocumentInputs(model: DocumentModel,documents: readonly DocumentInput[],signal?: AbortSignal,+ /** When set, send this text in place of the documents' content. */+ stubText?: string,): Promise<unknown> {if (!documents.length || !payload || typeof payload !== "object") return payload;const p = payload as Record<string, unknown>;if (model.api === "pi-messages" && p.context && typeof p.context === "object") {- return { ...p, context: await withDocumentInputs(p.context, { ...model, api: "fallback" }, documents, signal) };+ return {+ ...p,+ context: await withDocumentInputs(p.context, { ...model, api: "fallback" }, documents, signal, stubText),+ };}const google = model.api?.startsWith("google-");let listKey = "messages";@@ -153,7 +157,10 @@ export async function withDocumentInputs(JSON.stringify(payload, (key, value) => (["data", "image_url", "file_data"].includes(key) ? "" : value)),);const remainingTokens = Math.max(0, (model.contextWindow ?? 128_000) - promptTokens - 8192);- const blocks = await documentBlocks(documents, model, Math.min(100_000, remainingTokens), signal);+ const blocks =+ stubText === undefined+ ? await documentBlocks(documents, model, Math.min(100_000, remainingTokens), signal)+ : [bareText ? { text: stubText } : { type: textType, text: stubText }];let original: unknown[] = [];if (Array.isArray(content)) original = content;else if (typeof content === "string") original = [{ type: textType, text: content }];@@ -173,7 +180,9 @@ export async function withDocumentInputs(],};const nativeRouter =- model.provider === "openrouter" && documents.some((document) => nativeDocumentFormat(model, document) === "chat");+ stubText === undefined &&+ model.provider === "openrouter" &&+ documents.some((document) => nativeDocumentFormat(model, document) === "chat");return {...p,[listKey]: items,diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex 6b5257267..233d0cf45 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -7,6 +7,8 @@ import {type SteerIntake,} from "./harness-shared.ts";import { withDocumentInputs, type DocumentModel } from "./document-inputs.ts";+import type { DocumentInput } from "../core/document-inputs.ts";+import { documentStub } from "../core/document-kinds.ts";import { gatewayModelsJson, gatewayModelsVersion } from "../model/gateway-models.ts";import { Type } from "typebox";import { mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs";@@ -930,6 +932,31 @@ function piAssistantFailure(session: AssistantTextSession): ProviderTurnError |/** Codes that retry the turn on the configured fallback model. */const FALLBACK_CODES = new Set(["refusal", "model_unavailable"]);+/** Codes that retry a request once without its inline documents (oversized or unreadable attachments). */+const DOCUMENT_REJECTION_CODES = new Set(["context_too_long", "bad_request"]);++function documentsDroppedNote(documents: readonly DocumentInput[]): string {+ return (+ `The model provider rejected an earlier request in this task while it carried these attachments inline ` ++ `(most likely they exceed the model's context), so they were removed from model input. ` ++ documents.map((document) => documentStub(document)).join(" ") ++ ` If the file matters to the request, tell the person it could not be read in full here and read only the parts you need.`+ );+}++const DOCUMENTS_DROPPED_RETRY_NOTE =+ "[system] The previous request failed while it carried inline attachments; they have been replaced with notes on how to read them from saved files. Continue with the request.";++function dropFailedAssistant(session: AgentSession, messagesBefore: number): void {+ const state = session.agent.state;+ for (let i = state.messages.length - 1; i >= messagesBefore; i--) {+ const m = state.messages[i] as { role?: string; stopReason?: string } | undefined;+ if (m?.role === "assistant" && m.stopReason === "error") {+ state.messages = [...state.messages.slice(0, i), ...state.messages.slice(i + 1)];+ break;+ }+ }+}export function piLastAssistantTextOrThrow(session: AssistantTextSession): string | undefined {const err = piAssistantFailure(session);@@ -1660,6 +1687,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {model as DocumentModel,ref.documents ?? [],ref.abortSignal,+ ref.documentsStubbed ? documentsDroppedNote(ref.documents ?? []) : undefined,);const guarded = guardOutputBudget(finalPayload, model);if (guarded.kind === "raised") {@@ -1912,6 +1940,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {).agent?.subscribe?.((event) => {const e = event as { type?: string; message?: unknown };if (e.type === "agent_start" && toolAbort.signal.aborted) entry.agentSession.agent.abort();+ if (e.type === "message_end") noteDocumentRejection(e.message);return e.type === "message_end" ? onMessageEnd(e.message) : undefined;});const unsubscribe = entry.agentSession.subscribe((event) => {@@ -2132,14 +2161,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {entry.ref.effortLevel =custom?.effortLevel ?? turn.runtime?.effortLevel ?? defaultInteractiveThinkingLevel(fallback);applyTurnEffort(entry.agentSession, entry.ref.effortLevel);- const state = entry.agentSession.agent.state;- for (let i = state.messages.length - 1; i >= messagesBefore; i--) {- const m = state.messages[i] as { role?: string; stopReason?: string } | undefined;- if (m?.role === "assistant" && m.stopReason === "error") {- state.messages = [...state.messages.slice(0, i), ...state.messages.slice(i + 1)];- break;- }- }+ dropFailedAssistant(entry.agentSession, messagesBefore);const outcome = await raceTurnWallClock(entry.agentSession.prompt(refusalFallbackNote(modelDisplayName(fromId!), modelDisplayName(fallbackId), refusal),@@ -2150,8 +2172,42 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {if (outcome !== "ok") throw new NonRetryableTurnError(refusal);return true;};- const recoverRefusedRound = async (): Promise<void> => {+ let documentRejection: ProviderTurnError | undefined;+ let documentRetried = false;+ const noteDocumentRejection = (message: unknown): void => {+ if (documentRejection || !entry.ref.documents?.length) return;+ const failed = message as AssistantMessage | undefined;+ if (failed?.role !== "assistant" || failed.stopReason !== "error") return;+ const failure = piAssistantFailure({ messages: [failed] } as unknown as AssistantTextSession);+ if (!failure || !DOCUMENT_REJECTION_CODES.has(failure.code)) return;+ documentRejection = failure;+ entry.ref.documentsStubbed = true;+ console.error(+ `[pi] request with ${entry.ref.documents.length} inline document(s) rejected (${failure.code}) — replacing them with stubs session=${turn.session.id}`,+ );+ };+ const retryWithoutDocuments = async (): Promise<boolean> => {+ if (!documentRejection || documentRetried || userAborted || turn.cancel?.aborted) return false;+ const lastAssistant = entry.agentSession.messages+ .slice(messagesBefore)+ .findLast((m) => (m as { role?: string }).role === "assistant") as AssistantMessage | undefined;+ if (lastAssistant && lastAssistant.stopReason !== "error") return false;+ const capMs = raceCapMs();+ if (turnWallClockMs > 0 && capMs < EMPTY_ENDING_MIN_BUDGET_MS) return false;+ documentRetried = true;+ dropFailedAssistant(entry.agentSession, messagesBefore);+ const outcome = await raceTurnWallClock(entry.agentSession.prompt(DOCUMENTS_DROPPED_RETRY_NOTE), {+ capMs,+ extendMs: extendCapMs,+ abort: () => entry.agentSession.abort(),+ });+ if (userAborted) return false;+ if (outcome !== "ok") throw new NonRetryableTurnError(documentRejection.message);+ return true;+ };+ const recoverFailedRound = async (): Promise<void> => {if (entry.ref.runtimeHandoff || userAborted || turn.cancel?.aborted) return;+ await retryWithoutDocuments();const refusal = providerRefusalError(entry.agentSession, messagesBefore);if (!refusal) return;try {@@ -2175,7 +2231,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {abort: () => entry.agentSession.abort(),},);- if (wallClock === "ok") await recoverRefusedRound();+ if (wallClock === "ok") await recoverFailedRound();const goalAfterPrompt = entry.ref.goal;if (wallClock === "ok" &&@@ -2223,7 +2279,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {extendMs: extendCapMs,abort: () => entry.agentSession.abort(),});- if (outcome === "ok") await recoverRefusedRound();+ if (outcome === "ok") await recoverFailedRound();return outcome;},});@@ -2278,7 +2334,12 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {if (!userAborted && !cancelAbortRejection) {const turnErr = piTurnError(entry.agentSession, err, messagesBefore);let recovered = false;- if (turnErr instanceof ProviderTurnError && FALLBACK_CODES.has(turnErr.code)) {+ try {+ recovered = await retryWithoutDocuments();+ } catch (e) {+ swallow("pi: document-free retry", e);+ }+ if (!recovered && turnErr instanceof ProviderTurnError && FALLBACK_CODES.has(turnErr.code)) {try {recovered = await attemptRefusalFallback(turnErr.message);} catch (e) {diff --git a/src/sessions/session-store.ts b/src/sessions/session-store.tsindex ed8d1ddb5..ff7d8a180 100644--- a/src/sessions/session-store.ts+++ b/src/sessions/session-store.ts@@ -17,10 +17,17 @@ export function promptEnvelopeBody(envelope: unknown): { hash: string; body: strconst CONTEXT_SUMMARY_KIND = "context_summary";+export interface SummarizedDocument {+ name: string;+ artifactId: string;+}+export interface ContextSummaryPayload {kind: typeof CONTEXT_SUMMARY_KIND;throughSeq: number;text: string;+ /** Documents attached in the summarized turns; they are no longer replayed into model input. */+ documents?: SummarizedDocument[];}export function contextSummaryPayload(entry: SessionEntry): ContextSummaryPayload | null {@@ -31,13 +38,28 @@ export function contextSummaryPayload(entry: SessionEntry): ContextSummaryPayloatypeof payload.throughSeq === "number" &&typeof payload.text === "string") {- return { kind: CONTEXT_SUMMARY_KIND, throughSeq: payload.throughSeq, text: payload.text };+ const documents = Array.isArray(payload.documents)+ ? payload.documents.filter(+ (document): document is SummarizedDocument =>+ typeof document?.name === "string" && typeof document.artifactId === "string",+ )+ : [];+ return {+ kind: CONTEXT_SUMMARY_KIND,+ throughSeq: payload.throughSeq,+ text: payload.text,+ ...(documents.length ? { documents } : {}),+ };}return null;}-export function createContextSummaryPayload(throughSeq: number, text: string): ContextSummaryPayload {- return { kind: CONTEXT_SUMMARY_KIND, throughSeq, text };+export function createContextSummaryPayload(+ throughSeq: number,+ text: string,+ documents: readonly SummarizedDocument[] = [],+): ContextSummaryPayload {+ return { kind: CONTEXT_SUMMARY_KIND, throughSeq, text, ...(documents.length ? { documents: [...documents] } : {}) };}export interface ContextWindow {diff --git a/test/document-context-budget.test.ts b/test/document-context-budget.test.tsnew file mode 100644index 000000000..b4ec8611f--- /dev/null+++ b/test/document-context-budget.test.ts@@ -0,0 +1,90 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import {+ estimateEntryTokens,+ overBudgetFraction,+ planCompaction,+ summarizedDocuments,+ withSummarizedDocuments,+} from "../src/harness/context-compaction.ts";+import { estimateDocumentTokens } from "../src/core/document-kinds.ts";+import { createContextSummaryPayload } from "../src/sessions/session-store.ts";+import type { ScopeId, SessionEntry } from "../src/types.ts";++let seq = 0;+function entry(type: SessionEntry["type"], payload: unknown): SessionEntry {+ seq += 1;+ return {+ sessionId: "budget",+ seq,+ parentSeq: null,+ type,+ payload,+ scopeLabel: "personal:tester" as ScopeId,+ createdAt: seq,+ };+}++const pdf = (name: string, artifactId: string, sizeBytes = 1_000_000) => ({+ name,+ mimetype: "application/pdf",+ sizeBytes,+ direction: "in" as const,+ artifactId,+});++test("a user entry's document attachments count at their expanded size", () => {+ const plain = entry("user", { text: "please review" });+ const withPdf = entry("user", { text: "please review", attachments: [pdf("contract.pdf", "f1")] });+ assert.ok(estimateEntryTokens(withPdf) - estimateEntryTokens(plain) >= 250_000);+ const outbound = entry("user", {+ text: "please review",+ attachments: [{ ...pdf("contract.pdf", "f1"), direction: "out" }],+ });+ assert.equal(estimateEntryTokens(outbound), estimateEntryTokens(plain));+ assert.ok(estimateDocumentTokens({ name: "notes.txt", mimetype: "text/plain", sizeBytes: 50_000_000 }) <= 50_000);+});++test("history that is small in words but carries large PDFs compacts them out of the kept window", () => {+ const budget = 1_000_000;+ const history = [+ entry("user", { text: "here are the agreements", attachments: [pdf("a.pdf", "fa"), pdf("b.pdf", "fb")] }),+ entry("assistant", { text: "Read both." }),+ entry("user", { text: "and the amendment", attachments: [pdf("c.pdf", "fc")] }),+ entry("assistant", { text: "Read it." }),+ entry("user", { text: "and the exhibits", attachments: [pdf("d.pdf", "fd"), pdf("e.pdf", "fe")] }),+ entry("assistant", { text: "Read them." }),+ ];+ assert.ok(overBudgetFraction(history, budget, 0.9), "five 1MB PDFs no longer look like a few hundred tokens");+ const plan = planCompaction(history, budget, 0.6);+ assert.ok(plan);+ const keptDocuments = summarizedDocuments(plan.kept).length;+ assert.ok(keptDocuments * 250_000 <= budget * 0.6, `kept ${keptDocuments} documents`);+ assert.deepEqual(+ summarizedDocuments(plan.toSummarize).map((document) => document.name),+ ["a.pdf", "b.pdf", "c.pdf", "d.pdf", "e.pdf"].slice(0, 5 - keptDocuments),+ );+});++test("summaries list compacted documents once and carry them forward", () => {+ const first = summarizedDocuments([+ entry("user", { text: "x", attachments: [pdf("a.pdf", "fa"), pdf("b.pdf", "fb")] }),+ ]);+ const text = withSummarizedDocuments("Earlier: reviewed two agreements.", first);+ assert.match(text, /"a\.pdf" \(file fa\)/);+ assert.match(text, /GET \$AGENT_API_URL\/v1\/files\/<file id>\/content/);+ assert.equal(withSummarizedDocuments(text, first), text);+ const prior = entry("system", createContextSummaryPayload(1, text, first));+ const carried = summarizedDocuments([+ prior,+ entry("user", { text: "y", attachments: [pdf("c.pdf", "fc"), pdf("a.pdf", "fa")] }),+ ]);+ assert.deepEqual(+ carried.map((document) => document.artifactId),+ ["fb", "fc", "fa"],+ );+ const next = withSummarizedDocuments(`New summary.\n\n${text}`, carried);+ assert.equal(next.match(/no longer included inline/g)?.length, 1);+ assert.match(next, /"c\.pdf" \(file fc\)/);+ assert.equal(withSummarizedDocuments(next, []), "New summary.\n\nEarlier: reviewed two agreements.");+});diff --git a/test/document-inputs.test.ts b/test/document-inputs.test.tsindex a71aa4e1c..4f76d9d38 100644--- a/test/document-inputs.test.ts+++ b/test/document-inputs.test.ts@@ -4,10 +4,10 @@ import { test } from "node:test";import { createHash } from "node:crypto";import { Readable } from "node:stream";import { documentBlocks, nativeDocumentFormat, withDocumentInputs } from "../src/harness/document-inputs.ts";+import { isDocumentAttachment } from "../src/core/document-kinds.ts";import {documentFallbackText,documentText,- isDocumentAttachment,historicalDocumentMetas,loadDocumentInputs,type DocumentInput,@@ -258,7 +258,7 @@ test("steered reuploads already in the turn do not consume another document slot0,initial.documents,);- assert.deepEqual(steered, { documents: [], notices: [] });+ assert.deepEqual(steered, { documents: [], notices: [], tokens: 0 });assert.deepEqual(checked, ["steered-upload"]);assert.equal(opens(), 1);});@@ -281,7 +281,7 @@ test("document restoration checks current access and budgets before opening byteassert.equal(opens(), 0);const result = await loadDocumentInputs(store, [meta], () => true, 1);assert.equal(result.documents.length, 0);- assert.match(result.notices[0]!, /budget/);+ assert.match(result.notices[0]!, /not included inline/);assert.equal(opens(), 0);});diff --git a/test/orchestrator-documents.test.ts b/test/orchestrator-documents.test.tsindex 0197a2d11..75647d386 100644--- a/test/orchestrator-documents.test.ts+++ b/test/orchestrator-documents.test.ts@@ -169,3 +169,40 @@ for (const extension of ["docx", "pdf"]) {assert.doesNotMatch(JSON.stringify(followupCalls.at(-1)?.promptEnvelope), /"documents":/);});}++test("large documents never push a follow-up turn past the context budget", async () => {+ const budget = 20_000;+ const built = freshApp(budget);+ const request = {+ surface: "test" as const,+ actor: { externalId: "U1" },+ conversation: { kind: "dm" as const, threadRef: "dm:U1:large-docs" },+ };+ const sizes = new Map<string, number>();+ for (const name of ["one.txt", "two.txt", "three.txt"]) {+ const blob = await built.blobTransfer.put(Buffer.from(`${name} `.repeat(4_000)));+ sizes.set(name, blob.sizeBytes);+ const turn = await built.app.turn({+ ...request,+ text: `read ${name}`,+ attachments: [{ name, mimetype: "text/plain", sizeBytes: blob.sizeBytes, blobId: blob.blobId }],+ });+ assert.equal(turn.status, "ok");+ }+ const followup = await built.app.turn({ ...request, text: "compare the three files" });+ assert.equal(followup.status, "ok");+ const calls = (await built.sessions.listLlmRequests(followup.sessionId!)).filter((r) => r.model === "mock");+ const replayed = (+ (calls.at(-1)?.promptEnvelope as { documents?: Array<{ name: string }> } | undefined)?.documents ?? []+ ).map((document) => document.name);+ const replayedTokens = replayed.reduce((sum, name) => sum + Math.ceil(sizes.get(name)! / 4), 0);+ assert.ok(replayedTokens <= budget / 2, `replayed ${replayed.join(", ")} (~${replayedTokens} tokens)`);+ const entries = await built.sessions.getEntries(followup.sessionId!);+ const visible = JSON.stringify(entries.filter((entry) => entry.type === "system" || entry.type === "user").at(-1));+ for (const name of sizes.keys())+ if (!replayed.includes(name))+ assert.ok(+ JSON.stringify(entries.slice(-12)).includes(name),+ `${name} must stay discoverable as a stub after it leaves model context: ${visible.slice(0, 200)}`,+ );+});diff --git a/test/pi-harness-steering.test.ts b/test/pi-harness-steering.test.tsindex 2b95cd5a6..a20445aae 100644--- a/test/pi-harness-steering.test.ts+++ b/test/pi-harness-steering.test.ts@@ -623,3 +623,38 @@ for (const abort of ["shutdown", "stop"] as const) {}});}++test("a request rejected as too long retries once without its inline documents instead of failing the turn", async () => {+ const harness = createPiHarness({ apiKey: "sk-test" });+ const sink: Sink = { entries: [] };+ const realFetch = globalThis.fetch;+ const requests: string[] = [];+ const dataBase64 = (await readFile(new URL("./fixtures/documents/sample.pdf", import.meta.url))).toString("base64");+ globalThis.fetch = (async (_url: unknown, init?: RequestInit) => {+ const body = String(init?.body);+ requests.push(body);+ if (body.includes(dataBase64))+ return new Response(+ JSON.stringify({+ type: "error",+ error: { type: "invalid_request_error", message: "prompt is too long: 1320000 tokens > 1000000 maximum" },+ }),+ { status: 400, headers: { "content-type": "application/json" } },+ );+ return sse(textReplyEvents("answered without the inline file"));+ }) as typeof fetch;+ try {+ const result = await harness.turns.runTurn(+ turnInput("oversized-documents", sink, {+ documents: [{ name: "contract.pdf", mimeType: "application/pdf", dataBase64, artifactId: "doc-big" }],+ }),+ );+ assert.equal(result.reply, "answered without the inline file");+ assert.ok(requests[0]!.includes(dataBase64));+ assert.equal(requests.filter((body) => body.includes(dataBase64)).length, 1, "the document is sent only once");+ assert.match(requests.at(-1)!, /contract\.pdf/);+ assert.match(requests.at(-1)!, /doc-big/);+ } finally {+ globalThis.fetch = realFetch;+ }+});