Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

fix: report output truncation with token limit and usage (#1091)

Hary Ahujabe2bb8738d9b
diff --git a/.changeset/output-truncated-error.md b/.changeset/output-truncated-error.md
new file mode 100644
index 000000000..2dfa38126
--- /dev/null
+++ b/.changeset/output-truncated-error.md
@@ -0,0 +1,5 @@
+---
+'@truefoundry/trueforge-core': patch
+---
+
+When a model stops at its output token limit, fail the turn with `Output truncated: hit the <N> token limit. Raise max_tokens and retry.` (or "the provider's default token limit" when none was sent) instead of `max_tokens breached`. The tokens used are included when they differ from the limit.
diff --git a/packages/trueforge-core/src/core/llm/VercelAILLM.ts b/packages/trueforge-core/src/core/llm/VercelAILLM.ts
index 71cfdc8d1..397cebc81 100644
--- a/packages/trueforge-core/src/core/llm/VercelAILLM.ts
+++ b/packages/trueforge-core/src/core/llm/VercelAILLM.ts
@@ -45,6 +45,7 @@ import {
type RawAssistantMessageWithUsage,
type ThinkingBlock,
} from './LLMTypes';
+import { resolveMaxOutputTokens } from './usage';
// ---------------------------------------------------------------------------
// Public config
@@ -578,14 +579,6 @@ export function buildProviderOptions({
// Request conversion (OpenAI → SDK)
// ---------------------------------------------------------------------------
-/** Prefers `max_completion_tokens`, falling back to the deprecated `max_tokens`. */
-export function resolveMaxOutputTokens(
- body: Pick<ChatCompletionCreateParamsStreaming, 'max_completion_tokens'>,
-): number | undefined {
- const limit: unknown = body.max_completion_tokens ?? Reflect.get(body, 'max_tokens');
- return typeof limit === 'number' ? limit : undefined;
-}
-
/** Parses the MIME type from a data URI (`data:<mime>;base64,...`). */
export function parseMimeFromDataUri(uri: string): string | undefined {
const match = /^data:([^;,]+)/.exec(uri);
diff --git a/packages/trueforge-core/src/core/llm/usage.ts b/packages/trueforge-core/src/core/llm/usage.ts
index ae20e4d84..079845015 100644
--- a/packages/trueforge-core/src/core/llm/usage.ts
+++ b/packages/trueforge-core/src/core/llm/usage.ts
@@ -1,3 +1,4 @@
+import type { LLMCreateParamsStreaming } from './ILLM';
import type { CompletionUsage } from './LLMTypes';
export function estimateTokensForString(s: string): number {
@@ -22,3 +23,11 @@ export function mergeUsage(a: CompletionUsage, b: CompletionUsage): CompletionUs
cost_in_usd: sumOptional(a.cost_in_usd, b.cost_in_usd),
};
}
+
+/** Prefers `max_completion_tokens`, falling back to the deprecated `max_tokens`. */
+export function resolveMaxOutputTokens(
+ body: Pick<LLMCreateParamsStreaming, 'max_completion_tokens'>,
+): number | undefined {
+ const limit: unknown = body.max_completion_tokens ?? Reflect.get(body, 'max_tokens');
+ return typeof limit === 'number' ? limit : undefined;
+}
diff --git a/packages/trueforge-core/src/core/runtime/AgentThread.ts b/packages/trueforge-core/src/core/runtime/AgentThread.ts
index 978caaaaa..a8338ae85 100644
--- a/packages/trueforge-core/src/core/runtime/AgentThread.ts
+++ b/packages/trueforge-core/src/core/runtime/AgentThread.ts
@@ -49,7 +49,7 @@ import {
import { ModelSchema, type Model } from '../llm/model';
import { toOpenAIResponseFormat } from '../llm/responseFormat';
import { toOpenAIChatMessage } from '../llm/toOpenAIChatMessage';
-import { estimateTokensForString } from '../llm/usage';
+import { estimateTokensForString, resolveMaxOutputTokens } from '../llm/usage';
import { convertMCPServersToTools, type ConvertToolsResult, type MappedMCPTool } from '../mcp/convertMCPServers';
import { executeToolCalls } from '../mcp/executeToolCalls';
import type { IToolSet, MCPAuthRequired } from '../mcp/IMCPServer';
@@ -95,6 +95,7 @@ import {
toToolCallInfo,
} from './contextUtils';
import { DeferredTool } from './DeferredTool';
+import { finishReasonErrorMessage } from './finishReasonError';
import { createEmptyAgentThreadMetrics, updateMetricsFromUsage, type AgentThreadMetrics } from './metrics';
import { getClosableOpenToolCallIds, OpenToolCallCloser } from './OpenToolCallCloser';
import { isEmptyMessageContent, processAgentUserInput, type AgentInputUserMessage } from './UserInputMessage';
@@ -1187,13 +1188,16 @@ export class AgentThread {
model: modelFromDefinition(this.definition),
});
+ const finishError = finishReasonErrorMessage({
+ finishReason,
+ outputTokens: result.value.usage.output_tokens,
+ maxOutputTokens: resolveMaxOutputTokens(requestBody),
+ });
+
let completion: SubAgentCompletion | undefined;
if (this.parent) {
- if (finishReason === 'length') {
- const errorMessage = assistantMessageContentToStringForSubAgent(
- assistantMessage.content,
- 'max_tokens breached',
- );
+ if (finishError !== undefined) {
+ const errorMessage = assistantMessageContentToStringForSubAgent(assistantMessage.content, finishError);
completion = {
type: 'error',
output: agentAssistantMessage,
@@ -1218,8 +1222,8 @@ export class AgentThread {
completion,
});
- if (finishReason === 'length') {
- const errorContent = completion?.type === 'error' ? completion.error_message : 'max_tokens breached';
+ if (finishError !== undefined) {
+ const errorContent = completion?.type === 'error' ? completion.error_message : finishError;
yield this.generateErrorEvent(errorContent, agentAssistantMessage);
return { outcome: 'exit', modelMessageEventId };
}
diff --git a/packages/trueforge-core/src/core/runtime/finishReasonError.ts b/packages/trueforge-core/src/core/runtime/finishReasonError.ts
new file mode 100644
index 000000000..ffdfc81db
--- /dev/null
+++ b/packages/trueforge-core/src/core/runtime/finishReasonError.ts
@@ -0,0 +1,29 @@
+import type { FinishReason } from '../llm/LLMTypes';
+
+/**
+ * User-facing error for finish reasons that should fail the turn; undefined lets it continue or finish.
+ * Messages follow `<Headline>: <what happened>. <what the user can do>.`
+ */
+export function finishReasonErrorMessage({
+ finishReason,
+ outputTokens,
+ maxOutputTokens,
+}: {
+ finishReason: FinishReason | null;
+ outputTokens: number;
+ maxOutputTokens: number | undefined;
+}): string | undefined {
+ switch (finishReason) {
+ case 'length': {
+ const limit =
+ maxOutputTokens !== undefined
+ ? `the ${String(maxOutputTokens)} token limit`
+ : "the provider's default token limit";
+ // 0 means the provider omitted usage; a count equal to the limit adds nothing.
+ const used = outputTokens > 0 && outputTokens !== maxOutputTokens ? ` after ${String(outputTokens)} tokens` : '';
+ return `Output truncated: hit ${limit}${used}. Raise max_tokens and retry.`;
+ }
+ default:
+ return undefined;
+ }
+}
diff --git a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts
index 5b7766a0e..8184776cd 100644
--- a/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts
+++ b/packages/trueforge-core/tests/core/llm/VercelAILLM.model.test.ts
@@ -5,12 +5,8 @@
* exactOptionalPropertyTypes).
*/
import type { VercelAIProviderConfig, VercelAIProviderName } from '../../../src/core/llm/VercelAILLM';
-import {
- buildLanguageModel,
- buildStreamTextArgs,
- convertTools,
- resolveMaxOutputTokens,
-} from '../../../src/core/llm/VercelAILLM';
+import { buildLanguageModel, buildStreamTextArgs, convertTools } from '../../../src/core/llm/VercelAILLM';
+import { resolveMaxOutputTokens } from '../../../src/core/llm/usage';
function makeConfig(
overrides: Omit<Partial<VercelAIProviderConfig>, 'provider' | 'model'> & {
diff --git a/packages/trueforge-core/tests/core/runtime/finishReasonError.test.ts b/packages/trueforge-core/tests/core/runtime/finishReasonError.test.ts
new file mode 100644
index 000000000..7b263cfa9
--- /dev/null
+++ b/packages/trueforge-core/tests/core/runtime/finishReasonError.test.ts
@@ -0,0 +1,98 @@
+import type { ILLM } from '../../../src/core/llm/ILLM';
+import type {
+ ExtendedChatCompletionChunk,
+ FinishReason,
+ RawAssistantMessageWithUsage,
+} from '../../../src/core/llm/LLMTypes';
+import { AgentThread } from '../../../src/core/runtime/AgentThread';
+import { InternalEventType, type InternalThreadDoneEvent } from '../../../src/core/runtime/AgentThread.types';
+import { NOOP_AGENT_TRACING } from '../../../src/core/tracing/NoopAgentTracing';
+import '../harnessMocks';
+import { makeSilentLogger, TEST_MODEL_NAME } from '../harnessMocks';
+
+const silentLogger = makeSilentLogger();
+const OUTPUT_TOKENS = 4096;
+
+// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O
+async function* streamEndingWith(
+ finishReason: FinishReason,
+): AsyncGenerator<ExtendedChatCompletionChunk, RawAssistantMessageWithUsage, unknown> {
+ yield {
+ id: 'chunk-1',
+ object: 'chat.completion.chunk',
+ created: 0,
+ model: 'test-model',
+ choices: [
+ { index: 0, delta: { content: 'partial', role: 'assistant' }, finish_reason: finishReason, logprobs: null },
+ ],
+ };
+ return {
+ output: { role: 'assistant', content: 'partial' },
+ usage: { input_tokens: 10, output_tokens: OUTPUT_TOKENS, total_tokens: 10 + OUTPUT_TOKENS },
+ finish_reason: finishReason,
+ };
+}
+
+async function runUntilDone({
+ finishReason,
+ modelParams,
+}: {
+ finishReason: FinishReason;
+ modelParams: Record<string, unknown> | undefined;
+}): Promise<InternalThreadDoneEvent> {
+ const modelClient: ILLM = {
+ create: jest.fn().mockImplementation(() => streamEndingWith(finishReason)),
+ createNonStream: jest.fn(),
+ };
+ const thread = new AgentThread({
+ threadId: 'main',
+ title: 'Main',
+ tracing: NOOP_AGENT_TRACING,
+ logger: silentLogger,
+ definition: {
+ modelClient,
+ modelName: TEST_MODEL_NAME,
+ instruction: 'test',
+ toolSets: [],
+ ...(modelParams !== undefined ? { modelParams } : {}),
+ },
+ context: [{ role: 'user', content: 'hi' }],
+ });
+ for await (const event of thread.execute({ signal: new AbortController().signal })) {
+ if (event.type === InternalEventType.AGENT_DONE) {
+ return event;
+ }
+ }
+ throw new Error('thread finished without a done event');
+}
+
+describe('AgentThread finish reason errors', () => {
+ it.each<{
+ name: string;
+ finishReason: FinishReason;
+ modelParams: Record<string, unknown> | undefined;
+ error: string;
+ }>([
+ {
+ name: 'length at the configured limit',
+ finishReason: 'length',
+ modelParams: { max_tokens: OUTPUT_TOKENS },
+ error: 'Output truncated: hit the 4096 token limit. Raise max_tokens and retry.',
+ },
+ {
+ name: 'length below a different configured limit',
+ finishReason: 'length',
+ modelParams: { max_tokens: 16000 },
+ error: 'Output truncated: hit the 16000 token limit after 4096 tokens. Raise max_tokens and retry.',
+ },
+ {
+ name: 'length with no configured limit',
+ finishReason: 'length',
+ modelParams: undefined,
+ error: "Output truncated: hit the provider's default token limit after 4096 tokens. Raise max_tokens and retry.",
+ },
+ ])('fails the turn for $name', async ({ finishReason, modelParams, error }) => {
+ const done = await runUntilDone({ finishReason, modelParams });
+ expect(done).toMatchObject({ status: 'error', error });
+ });
+});