The open-source agent harness - the runtime layer that turns an LLM into a working agent.
fix: report output truncation with token limit and usage (#1091)
Hary Ahujabe2bb8738d9b
diff --git a/.changeset/output-truncated-error.md b/.changeset/output-truncated-error.mdnew file mode 100644index 000000000..2dfa38126--- /dev/null+++ b/.changeset/output-truncated-error.md@@ -0,0 +1,5 @@+---+'@truefoundry/trueforge-core': patch+---++When a model stops at its output token limit, fail the turn with `Output truncated: hit the <N> token limit. Raise max_tokens and retry.` (or "the provider's default token limit" when none was sent) instead of `max_tokens breached`. The tokens used are included when they differ from the limit.diff --git a/packages/trueforge-core/src/core/llm/VercelAILLM.ts b/packages/trueforge-core/src/core/llm/VercelAILLM.tsindex 71cfdc8d1..397cebc81 100644--- a/packages/trueforge-core/src/core/llm/VercelAILLM.ts+++ b/packages/trueforge-core/src/core/llm/VercelAILLM.ts@@ -45,6 +45,7 @@ import {type RawAssistantMessageWithUsage,type ThinkingBlock,} from './LLMTypes';+import { resolveMaxOutputTokens } from './usage';// ---------------------------------------------------------------------------// Public config@@ -578,14 +579,6 @@ export function buildProviderOptions({// Request conversion (OpenAI → SDK)// ----------------------------------------------------------------------------/** Prefers `max_completion_tokens`, falling back to the deprecated `max_tokens`. */-export function resolveMaxOutputTokens(- body: Pick<ChatCompletionCreateParamsStreaming, 'max_completion_tokens'>,-): number | undefined {- const limit: unknown = body.max_completion_tokens ?? Reflect.get(body, 'max_tokens');- return typeof limit === 'number' ? limit : undefined;-}-/** Parses the MIME type from a data URI (`data:<mime>;base64,...`). */export function parseMimeFromDataUri(uri: string): string | undefined {const match = /^data:([^;,]+)/.exec(uri);diff --git a/packages/trueforge-core/src/core/llm/usage.ts b/packages/trueforge-core/src/core/llm/usage.tsindex ae20e4d84..079845015 100644--- a/packages/trueforge-core/src/core/llm/usage.ts+++ b/packages/trueforge-core/src/core/llm/usage.ts@@ -1,3 +1,4 @@+import type { LLMCreateParamsStreaming } from './ILLM';import type { CompletionUsage } from './LLMTypes';export function estimateTokensForString(s: string): number {@@ -22,3 +23,11 @@ export function mergeUsage(a: CompletionUsage, b: CompletionUsage): CompletionUscost_in_usd: sumOptional(a.cost_in_usd, b.cost_in_usd),};}++/** Prefers `max_completion_tokens`, falling back to the deprecated `max_tokens`. */+export function resolveMaxOutputTokens(+ body: Pick<LLMCreateParamsStreaming, 'max_completion_tokens'>,+): number | undefined {+ const limit: unknown = body.max_completion_tokens ?? Reflect.get(body, 'max_tokens');+ return typeof limit === 'number' ? limit : undefined;+}diff --git a/packages/trueforge-core/src/core/runtime/AgentThread.ts b/packages/trueforge-core/src/core/runtime/AgentThread.tsindex 978caaaaa..a8338ae85 100644--- a/packages/trueforge-core/src/core/runtime/AgentThread.ts+++ b/packages/trueforge-core/src/core/runtime/AgentThread.ts@@ -49,7 +49,7 @@ import {import { ModelSchema, type Model } from '../llm/model';import { toOpenAIResponseFormat } from '../llm/responseFormat';import { toOpenAIChatMessage } from '../llm/toOpenAIChatMessage';-import { estimateTokensForString } from '../llm/usage';+import { estimateTokensForString, resolveMaxOutputTokens } from '../llm/usage';import { convertMCPServersToTools, type ConvertToolsResult, type MappedMCPTool } from '../mcp/convertMCPServers';import { executeToolCalls } from '../mcp/executeToolCalls';import type { IToolSet, MCPAuthRequired } from '../mcp/IMCPServer';@@ -95,6 +95,7 @@ import {toToolCallInfo,} from './contextUtils';import { DeferredTool } from './DeferredTool';+import { finishReasonErrorMessage } from './finishReasonError';import { createEmptyAgentThreadMetrics, updateMetricsFromUsage, type AgentThreadMetrics } from './metrics';import { getClosableOpenToolCallIds, OpenToolCallCloser } from './OpenToolCallCloser';import { isEmptyMessageContent, processAgentUserInput, type AgentInputUserMessage } from './UserInputMessage';@@ -1187,13 +1188,16 @@ export class AgentThread {model: modelFromDefinition(this.definition),});+ const finishError = finishReasonErrorMessage({+ finishReason,+ outputTokens: result.value.usage.output_tokens,+ maxOutputTokens: resolveMaxOutputTokens(requestBody),+ });+let completion: SubAgentCompletion | undefined;if (this.parent) {- if (finishReason === 'length') {- const errorMessage = assistantMessageContentToStringForSubAgent(- assistantMessage.content,- 'max_tokens breached',- );+ if (finishError !== undefined) {+ const errorMessage = assistantMessageContentToStringForSubAgent(assistantMessage.content, finishError);completion = {type: 'error',output: agentAssistantMessage,@@ -1218,8 +1222,8 @@ export class AgentThread {completion,});- if (finishReason === 'length') {- const errorContent = completion?.type === 'error' ? completion.error_message : 'max_tokens breached';+ if (finishError !== undefined) {+ const errorContent = completion?.type === 'error' ? completion.error_message : finishError;yield this.generateErrorEvent(errorContent, agentAssistantMessage);return { outcome: 'exit', modelMessageEventId };}diff --git a/packages/trueforge-core/src/core/runtime/finishReasonError.ts b/packages/trueforge-core/src/core/runtime/finishReasonError.tsnew file mode 100644index 000000000..ffdfc81db--- /dev/null+++ b/packages/trueforge-core/src/core/runtime/finishReasonError.ts@@ -0,0 +1,29 @@+import type { FinishReason } from '../llm/LLMTypes';++/**+ * User-facing error for finish reasons that should fail the turn; undefined lets it continue or finish.+ * Messages follow `<Headline>: <what happened>. <what the user can do>.`+ */+export function finishReasonErrorMessage({+ finishReason,+ outputTokens,+ maxOutputTokens,+}: {+ finishReason: FinishReason | null;+ outputTokens: number;+ maxOutputTokens: number | undefined;+}): string | undefined {+ switch (finishReason) {+ case 'length': {+ const limit =+ maxOutputTokens !== undefined+ ? `the ${String(maxOutputTokens)} token limit`+ : "the provider's default token limit";+ // 0 means the provider omitted usage; a count equal to the limit adds nothing.+ const used = outputTokens > 0 && outputTokens !== maxOutputTokens ? ` after ${String(outputTokens)} tokens` : '';+ return `Output truncated: hit ${limit}${used}. Raise max_tokens and retry.`;+ }+ default:+ return undefined;+ }+}diff --git a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.tsindex 5b7766a0e..8184776cd 100644--- a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts+++ b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts@@ -5,12 +5,8 @@* exactOptionalPropertyTypes).*/import type { VercelAIProviderConfig, VercelAIProviderName } from '../../../src/core/llm/VercelAILLM';-import {- buildLanguageModel,- buildStreamTextArgs,- convertTools,- resolveMaxOutputTokens,-} from '../../../src/core/llm/VercelAILLM';+import { buildLanguageModel, buildStreamTextArgs, convertTools } from '../../../src/core/llm/VercelAILLM';+import { resolveMaxOutputTokens } from '../../../src/core/llm/usage';function makeConfig(overrides: Omit<Partial<VercelAIProviderConfig>, 'provider' | 'model'> & {diff --git a/packages/trueforge-core/tests/core/runtime/finishReasonError.test.ts b/packages/trueforge-core/tests/core/runtime/finishReasonError.test.tsnew file mode 100644index 000000000..7b263cfa9--- /dev/null+++ b/packages/trueforge-core/tests/core/runtime/finishReasonError.test.ts@@ -0,0 +1,98 @@+import type { ILLM } from '../../../src/core/llm/ILLM';+import type {+ ExtendedChatCompletionChunk,+ FinishReason,+ RawAssistantMessageWithUsage,+} from '../../../src/core/llm/LLMTypes';+import { AgentThread } from '../../../src/core/runtime/AgentThread';+import { InternalEventType, type InternalThreadDoneEvent } from '../../../src/core/runtime/AgentThread.types';+import { NOOP_AGENT_TRACING } from '../../../src/core/tracing/NoopAgentTracing';+import '../harnessMocks';+import { makeSilentLogger, TEST_MODEL_NAME } from '../harnessMocks';++const silentLogger = makeSilentLogger();+const OUTPUT_TOKENS = 4096;++// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O+async function* streamEndingWith(+ finishReason: FinishReason,+): AsyncGenerator<ExtendedChatCompletionChunk, RawAssistantMessageWithUsage, unknown> {+ yield {+ id: 'chunk-1',+ object: 'chat.completion.chunk',+ created: 0,+ model: 'test-model',+ choices: [+ { index: 0, delta: { content: 'partial', role: 'assistant' }, finish_reason: finishReason, logprobs: null },+ ],+ };+ return {+ output: { role: 'assistant', content: 'partial' },+ usage: { input_tokens: 10, output_tokens: OUTPUT_TOKENS, total_tokens: 10 + OUTPUT_TOKENS },+ finish_reason: finishReason,+ };+}++async function runUntilDone({+ finishReason,+ modelParams,+}: {+ finishReason: FinishReason;+ modelParams: Record<string, unknown> | undefined;+}): Promise<InternalThreadDoneEvent> {+ const modelClient: ILLM = {+ create: jest.fn().mockImplementation(() => streamEndingWith(finishReason)),+ createNonStream: jest.fn(),+ };+ const thread = new AgentThread({+ threadId: 'main',+ title: 'Main',+ tracing: NOOP_AGENT_TRACING,+ logger: silentLogger,+ definition: {+ modelClient,+ modelName: TEST_MODEL_NAME,+ instruction: 'test',+ toolSets: [],+ ...(modelParams !== undefined ? { modelParams } : {}),+ },+ context: [{ role: 'user', content: 'hi' }],+ });+ for await (const event of thread.execute({ signal: new AbortController().signal })) {+ if (event.type === InternalEventType.AGENT_DONE) {+ return event;+ }+ }+ throw new Error('thread finished without a done event');+}++describe('AgentThread finish reason errors', () => {+ it.each<{+ name: string;+ finishReason: FinishReason;+ modelParams: Record<string, unknown> | undefined;+ error: string;+ }>([+ {+ name: 'length at the configured limit',+ finishReason: 'length',+ modelParams: { max_tokens: OUTPUT_TOKENS },+ error: 'Output truncated: hit the 4096 token limit. Raise max_tokens and retry.',+ },+ {+ name: 'length below a different configured limit',+ finishReason: 'length',+ modelParams: { max_tokens: 16000 },+ error: 'Output truncated: hit the 16000 token limit after 4096 tokens. Raise max_tokens and retry.',+ },+ {+ name: 'length with no configured limit',+ finishReason: 'length',+ modelParams: undefined,+ error: "Output truncated: hit the provider's default token limit after 4096 tokens. Raise max_tokens and retry.",+ },+ ])('fails the turn for $name', async ({ finishReason, modelParams, error }) => {+ const done = await runUntilDone({ finishReason, modelParams });+ expect(done).toMatchObject({ status: 'error', error });+ });+});