Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

Return OpenAI's reasoning, and replay it as one item per token (#83)

pranjaltrvd0471a5f7e2a5
diff --git a/packages/harness/src/core/llm/VercelAILLM.ts b/packages/harness/src/core/llm/VercelAILLM.ts
index bf4576c91..4d76716a4 100644
--- a/packages/harness/src/core/llm/VercelAILLM.ts
+++ b/packages/harness/src/core/llm/VercelAILLM.ts
@@ -351,12 +351,15 @@ function openaiProviderOptions({
const user = readBodyField({ rawBody, key: 'user' });
const promptCacheKey = readBodyField({ rawBody, key: 'prompt_cache_key' });
const parallelToolCalls = readBodyField({ rawBody, key: 'parallel_tool_calls' });
+ const reasoningSummary = readBodyField({ rawBody, key: 'reasoning_summary' });
// No server-side storage, and request the encrypted reasoning token so replay stays stateless.
return {
store: false,
include: ['reasoning.encrypted_content'],
+ reasoningSummary: 'auto',
// gpt-5.6 serves both xhigh and max, so the aliased top-level level asks for one rung less.
...maxEffortOption(reasoningEffort),
+ ...(reasoningSummary !== undefined ? { reasoningSummary } : {}),
...(strictJsonSchema !== undefined ? { strictJsonSchema } : {}),
...(serviceTier !== undefined ? { serviceTier } : {}),
...(user !== undefined ? { user } : {}),
@@ -977,6 +980,22 @@ function applyReasoningSignature({
}
}
+/**
+ * Which reasoning item a stream part belongs to. OpenAI is alone in splitting one item across
+ * several parts, so elsewhere this is absent and every part stands on its own.
+ */
+function reasoningItemId(providerMetadata: ProviderMetadata | undefined): string | undefined {
+ if (providerMetadata === undefined) {
+ return undefined;
+ }
+ for (const meta of Object.values(providerMetadata)) {
+ if ('itemId' in meta && typeof meta['itemId'] === 'string') {
+ return meta['itemId'];
+ }
+ }
+ return undefined;
+}
+
/**
* Where the SDK reports what it changed about our request: an effort a model cannot honour and so
* was coerced, a setting the provider ignored. Nothing else surfaces these.
@@ -1028,7 +1047,34 @@ export async function* mapStreamToChunks({
let nextToolIndex = 0;
let accumulatedText = '';
const accumulatedThinking: ThinkingBlock[] = [];
+ const thinkingByReasoningItem = new Map<string, ThinkingBlock>();
let currentThinkingBlock: ThinkingBlock | null = null;
+
+ /**
+ * The block this reasoning part belongs to: the one already open for its item, or a new one.
+ *
+ * OpenAI streams a single reasoning item as several summary parts, and hands every part the one
+ * token that identifies the item. A block per part would therefore replay that item once per
+ * part, every copy presenting the same token with a different piece of the reasoning. Grouping by
+ * item keeps one block per token, which is the only shape the provider can be replayed.
+ */
+ const openThinkingBlock = (providerMetadata: ProviderMetadata | undefined): ThinkingBlock => {
+ const itemId = reasoningItemId(providerMetadata);
+ const started = itemId !== undefined ? thinkingByReasoningItem.get(itemId) : undefined;
+ if (started !== undefined) {
+ // Each part is its own titled paragraph, and a part can be empty.
+ if (started.thinking.length > 0) {
+ started.thinking += '\n\n';
+ }
+ return started;
+ }
+ const block: ThinkingBlock = { type: 'thinking', thinking: '' };
+ accumulatedThinking.push(block);
+ if (itemId !== undefined) {
+ thinkingByReasoningItem.set(itemId, block);
+ }
+ return block;
+ };
let finalUsage: CompletionUsage = getEmptyUsage();
let finalFinishReason: RawAssistantMessageWithUsage['finish_reason'] = 'stop';
@@ -1051,18 +1097,14 @@ export async function* mapStreamToChunks({
}
case 'reasoning-start': {
- currentThinkingBlock = { type: 'thinking', thinking: '' };
- accumulatedThinking.push(currentThinkingBlock);
+ currentThinkingBlock = openThinkingBlock(part.providerMetadata);
break;
}
case 'reasoning-delta': {
// A delta with no preceding `reasoning-start` would otherwise drop the block and its
// signature, breaking replay while the text still streams out.
- if (currentThinkingBlock === null) {
- currentThinkingBlock = { type: 'thinking', thinking: '' };
- accumulatedThinking.push(currentThinkingBlock);
- }
+ currentThinkingBlock ??= openThinkingBlock(part.providerMetadata);
currentThinkingBlock.thinking += part.text;
applyReasoningSignature({ block: currentThinkingBlock, providerMetadata: part.providerMetadata });
yield {
diff --git a/packages/harness/tests/core/llm/VercelAILLM.output.test.ts b/packages/harness/tests/core/llm/VercelAILLM.output.test.ts
index a055e6937..5495c09c6 100644
--- a/packages/harness/tests/core/llm/VercelAILLM.output.test.ts
+++ b/packages/harness/tests/core/llm/VercelAILLM.output.test.ts
@@ -149,6 +149,26 @@ describe('buildProviderOptions', () => {
expect(opts['openai']).toMatchObject({ reasoningEffort: 'max' });
});
+ it('asks for a reasoning summary, without which the model reasons but returns no text', () => {
+ const opts = buildProviderOptions({
+ config,
+ reasoningEffort: undefined,
+ structuredOutputSpec: textSpec,
+ rawBody: {},
+ });
+ expect(opts['openai']).toMatchObject({ reasoningSummary: 'auto' });
+ });
+
+ it('lets an explicit reasoning_summary replace the default', () => {
+ const opts = buildProviderOptions({
+ config,
+ reasoningEffort: undefined,
+ structuredOutputSpec: textSpec,
+ rawBody: { reasoning_summary: 'detailed' },
+ });
+ expect(opts['openai']).toMatchObject({ reasoningSummary: 'detailed' });
+ });
+
it('includes strictJsonSchema:true when spec is json_schema with strict:true', () => {
const opts = buildProviderOptions({
config: config,
diff --git a/packages/harness/tests/core/llm/VercelAILLM.stream.test.ts b/packages/harness/tests/core/llm/VercelAILLM.stream.test.ts
index 659e3ee32..8a304b63f 100644
--- a/packages/harness/tests/core/llm/VercelAILLM.stream.test.ts
+++ b/packages/harness/tests/core/llm/VercelAILLM.stream.test.ts
@@ -204,6 +204,124 @@ describe('mapStreamToChunks', () => {
});
});
+ // OpenAI's real wire shape once summaries are on: one reasoning item arrives as several summary
+ // parts, each closed with the item's single token. A block per part would replay that one item
+ // once per part, all copies carrying the same token with the summary divided between them.
+ it('keeps the summary parts of one OpenAI reasoning item in a single block', async () => {
+ const itemId = 'rs_abc';
+ const { final } = await drainStream(
+ mapStreamToChunks({
+ stream: makeStream([
+ { type: 'reasoning-start', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },
+ { type: 'reasoning-delta', id: `${itemId}:0`, text: 'first summary' },
+ {
+ type: 'reasoning-end',
+ id: `${itemId}:0`,
+ providerMetadata: { openai: { itemId, reasoningEncryptedContent: 'enc-item' } },
+ },
+ { type: 'reasoning-start', id: `${itemId}:1`, providerMetadata: { openai: { itemId } } },
+ { type: 'reasoning-delta', id: `${itemId}:1`, text: 'second summary' },
+ {
+ type: 'reasoning-end',
+ id: `${itemId}:1`,
+ providerMetadata: { openai: { itemId, reasoningEncryptedContent: 'enc-item' } },
+ },
+ makeFinishStep('stop'),
+ ]),
+ chunkMeta: CHUNK_META,
+ }),
+ );
+
+ expect(final.output.thinking_blocks).toEqual([
+ { type: 'thinking', thinking: 'first summary\n\nsecond summary', signature: 'enc-item' },
+ ]);
+ });
+
+ // The same grouping has to hold when only the closing part carries the token, which is how the
+ // provider behaved before it began repeating it, and all a stream is contractually promised.
+ it('signs the whole item when only its closing part carries the token', async () => {
+ const itemId = 'rs_abc';
+ const { final } = await drainStream(
+ mapStreamToChunks({
+ stream: makeStream([
+ { type: 'reasoning-start', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },
+ { type: 'reasoning-delta', id: `${itemId}:0`, text: 'first summary' },
+ { type: 'reasoning-end', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },
+ { type: 'reasoning-start', id: `${itemId}:1`, providerMetadata: { openai: { itemId } } },
+ { type: 'reasoning-delta', id: `${itemId}:1`, text: 'second summary' },
+ {
+ type: 'reasoning-end',
+ id: `${itemId}:1`,
+ providerMetadata: { openai: { itemId, reasoningEncryptedContent: 'enc-sig' } },
+ },
+ makeFinishStep('stop'),
+ ]),
+ chunkMeta: CHUNK_META,
+ }),
+ );
+
+ expect(final.output.thinking_blocks).toEqual([
+ { type: 'thinking', thinking: 'first summary\n\nsecond summary', signature: 'enc-sig' },
+ ]);
+ });
+
+ // An item can open a summary part it never writes to, which must not push the reasoning that
+ // follows behind a blank line.
+ it('does not indent the text of an item whose first part is empty', async () => {
+ const itemId = 'rs_abc';
+ const { final } = await drainStream(
+ mapStreamToChunks({
+ stream: makeStream([
+ { type: 'reasoning-start', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },
+ { type: 'reasoning-end', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },
+ { type: 'reasoning-start', id: `${itemId}:1`, providerMetadata: { openai: { itemId } } },
+ { type: 'reasoning-delta', id: `${itemId}:1`, text: 'the only summary' },
+ {
+ type: 'reasoning-end',
+ id: `${itemId}:1`,
+ providerMetadata: { openai: { itemId, reasoningEncryptedContent: 'enc-item' } },
+ },
+ makeFinishStep('stop'),
+ ]),
+ chunkMeta: CHUNK_META,
+ }),
+ );
+
+ expect(final.output.thinking_blocks).toEqual([
+ { type: 'thinking', thinking: 'the only summary', signature: 'enc-item' },
+ ]);
+ });
+
+ it('keeps separate reasoning items in separate blocks, each with its own token', async () => {
+ const { final } = await drainStream(
+ mapStreamToChunks({
+ stream: makeStream([
+ { type: 'reasoning-start', id: 'rs_1:0', providerMetadata: { openai: { itemId: 'rs_1' } } },
+ { type: 'reasoning-delta', id: 'rs_1:0', text: 'before the tool call' },
+ {
+ type: 'reasoning-end',
+ id: 'rs_1:0',
+ providerMetadata: { openai: { itemId: 'rs_1', reasoningEncryptedContent: 'enc-1' } },
+ },
+ { type: 'reasoning-start', id: 'rs_2:0', providerMetadata: { openai: { itemId: 'rs_2' } } },
+ { type: 'reasoning-delta', id: 'rs_2:0', text: 'after the tool call' },
+ {
+ type: 'reasoning-end',
+ id: 'rs_2:0',
+ providerMetadata: { openai: { itemId: 'rs_2', reasoningEncryptedContent: 'enc-2' } },
+ },
+ makeFinishStep('stop'),
+ ]),
+ chunkMeta: CHUNK_META,
+ }),
+ );
+
+ expect(final.output.thinking_blocks).toEqual([
+ { type: 'thinking', thinking: 'before the tool call', signature: 'enc-1' },
+ { type: 'thinking', thinking: 'after the tool call', signature: 'enc-2' },
+ ]);
+ });
+
// Anthropic's real wire shape: signature_delta surfaces as a text-less reasoning-delta, not on
// reasoning-end. Missing it leaves the block unsigned, and Anthropic drops unsigned thinking
// blocks when they are replayed, silently losing the reasoning chain.