The open-source agent harness - the runtime layer that turns an LLM into a working agent.
Return OpenAI's reasoning, and replay it as one item per token (#83)
pranjaltrvd0471a5f7e2a5
diff --git a/packages/harness/src/core/llm/VercelAILLM.ts b/packages/harness/src/core/llm/VercelAILLM.tsindex bf4576c91..4d76716a4 100644--- a/packages/harness/src/core/llm/VercelAILLM.ts+++ b/packages/harness/src/core/llm/VercelAILLM.ts@@ -351,12 +351,15 @@ function openaiProviderOptions({const user = readBodyField({ rawBody, key: 'user' });const promptCacheKey = readBodyField({ rawBody, key: 'prompt_cache_key' });const parallelToolCalls = readBodyField({ rawBody, key: 'parallel_tool_calls' });+ const reasoningSummary = readBodyField({ rawBody, key: 'reasoning_summary' });// No server-side storage, and request the encrypted reasoning token so replay stays stateless.return {store: false,include: ['reasoning.encrypted_content'],+ reasoningSummary: 'auto',// gpt-5.6 serves both xhigh and max, so the aliased top-level level asks for one rung less....maxEffortOption(reasoningEffort),+ ...(reasoningSummary !== undefined ? { reasoningSummary } : {}),...(strictJsonSchema !== undefined ? { strictJsonSchema } : {}),...(serviceTier !== undefined ? { serviceTier } : {}),...(user !== undefined ? { user } : {}),@@ -977,6 +980,22 @@ function applyReasoningSignature({}}+/**+ * Which reasoning item a stream part belongs to. OpenAI is alone in splitting one item across+ * several parts, so elsewhere this is absent and every part stands on its own.+ */+function reasoningItemId(providerMetadata: ProviderMetadata | undefined): string | undefined {+ if (providerMetadata === undefined) {+ return undefined;+ }+ for (const meta of Object.values(providerMetadata)) {+ if ('itemId' in meta && typeof meta['itemId'] === 'string') {+ return meta['itemId'];+ }+ }+ return undefined;+}+/*** Where the SDK reports what it changed about our request: an effort a model cannot honour and so* was coerced, a setting the provider ignored. Nothing else surfaces these.@@ -1028,7 +1047,34 @@ export async function* mapStreamToChunks({let nextToolIndex = 0;let accumulatedText = '';const accumulatedThinking: ThinkingBlock[] = [];+ const thinkingByReasoningItem = new Map<string, ThinkingBlock>();let currentThinkingBlock: ThinkingBlock | null = null;++ /**+ * The block this reasoning part belongs to: the one already open for its item, or a new one.+ *+ * OpenAI streams a single reasoning item as several summary parts, and hands every part the one+ * token that identifies the item. A block per part would therefore replay that item once per+ * part, every copy presenting the same token with a different piece of the reasoning. Grouping by+ * item keeps one block per token, which is the only shape the provider can be replayed.+ */+ const openThinkingBlock = (providerMetadata: ProviderMetadata | undefined): ThinkingBlock => {+ const itemId = reasoningItemId(providerMetadata);+ const started = itemId !== undefined ? thinkingByReasoningItem.get(itemId) : undefined;+ if (started !== undefined) {+ // Each part is its own titled paragraph, and a part can be empty.+ if (started.thinking.length > 0) {+ started.thinking += '\n\n';+ }+ return started;+ }+ const block: ThinkingBlock = { type: 'thinking', thinking: '' };+ accumulatedThinking.push(block);+ if (itemId !== undefined) {+ thinkingByReasoningItem.set(itemId, block);+ }+ return block;+ };let finalUsage: CompletionUsage = getEmptyUsage();let finalFinishReason: RawAssistantMessageWithUsage['finish_reason'] = 'stop';@@ -1051,18 +1097,14 @@ export async function* mapStreamToChunks({}case 'reasoning-start': {- currentThinkingBlock = { type: 'thinking', thinking: '' };- accumulatedThinking.push(currentThinkingBlock);+ currentThinkingBlock = openThinkingBlock(part.providerMetadata);break;}case 'reasoning-delta': {// A delta with no preceding `reasoning-start` would otherwise drop the block and its// signature, breaking replay while the text still streams out.- if (currentThinkingBlock === null) {- currentThinkingBlock = { type: 'thinking', thinking: '' };- accumulatedThinking.push(currentThinkingBlock);- }+ currentThinkingBlock ??= openThinkingBlock(part.providerMetadata);currentThinkingBlock.thinking += part.text;applyReasoningSignature({ block: currentThinkingBlock, providerMetadata: part.providerMetadata });yield {diff --git a/packages/harness/tests/core/llm/VercelAILLM.output.test.ts b/packages/harness/tests/core/llm/VercelAILLM.output.test.tsindex a055e6937..5495c09c6 100644--- a/packages/harness/tests/core/llm/VercelAILLM.output.test.ts+++ b/packages/harness/tests/core/llm/VercelAILLM.output.test.ts@@ -149,6 +149,26 @@ describe('buildProviderOptions', () => {expect(opts['openai']).toMatchObject({ reasoningEffort: 'max' });});+ it('asks for a reasoning summary, without which the model reasons but returns no text', () => {+ const opts = buildProviderOptions({+ config,+ reasoningEffort: undefined,+ structuredOutputSpec: textSpec,+ rawBody: {},+ });+ expect(opts['openai']).toMatchObject({ reasoningSummary: 'auto' });+ });++ it('lets an explicit reasoning_summary replace the default', () => {+ const opts = buildProviderOptions({+ config,+ reasoningEffort: undefined,+ structuredOutputSpec: textSpec,+ rawBody: { reasoning_summary: 'detailed' },+ });+ expect(opts['openai']).toMatchObject({ reasoningSummary: 'detailed' });+ });+it('includes strictJsonSchema:true when spec is json_schema with strict:true', () => {const opts = buildProviderOptions({config: config,diff --git a/packages/harness/tests/core/llm/VercelAILLM.stream.test.ts b/packages/harness/tests/core/llm/VercelAILLM.stream.test.tsindex 659e3ee32..8a304b63f 100644--- a/packages/harness/tests/core/llm/VercelAILLM.stream.test.ts+++ b/packages/harness/tests/core/llm/VercelAILLM.stream.test.ts@@ -204,6 +204,124 @@ describe('mapStreamToChunks', () => {});});+ // OpenAI's real wire shape once summaries are on: one reasoning item arrives as several summary+ // parts, each closed with the item's single token. A block per part would replay that one item+ // once per part, all copies carrying the same token with the summary divided between them.+ it('keeps the summary parts of one OpenAI reasoning item in a single block', async () => {+ const itemId = 'rs_abc';+ const { final } = await drainStream(+ mapStreamToChunks({+ stream: makeStream([+ { type: 'reasoning-start', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },+ { type: 'reasoning-delta', id: `${itemId}:0`, text: 'first summary' },+ {+ type: 'reasoning-end',+ id: `${itemId}:0`,+ providerMetadata: { openai: { itemId, reasoningEncryptedContent: 'enc-item' } },+ },+ { type: 'reasoning-start', id: `${itemId}:1`, providerMetadata: { openai: { itemId } } },+ { type: 'reasoning-delta', id: `${itemId}:1`, text: 'second summary' },+ {+ type: 'reasoning-end',+ id: `${itemId}:1`,+ providerMetadata: { openai: { itemId, reasoningEncryptedContent: 'enc-item' } },+ },+ makeFinishStep('stop'),+ ]),+ chunkMeta: CHUNK_META,+ }),+ );++ expect(final.output.thinking_blocks).toEqual([+ { type: 'thinking', thinking: 'first summary\n\nsecond summary', signature: 'enc-item' },+ ]);+ });++ // The same grouping has to hold when only the closing part carries the token, which is how the+ // provider behaved before it began repeating it, and all a stream is contractually promised.+ it('signs the whole item when only its closing part carries the token', async () => {+ const itemId = 'rs_abc';+ const { final } = await drainStream(+ mapStreamToChunks({+ stream: makeStream([+ { type: 'reasoning-start', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },+ { type: 'reasoning-delta', id: `${itemId}:0`, text: 'first summary' },+ { type: 'reasoning-end', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },+ { type: 'reasoning-start', id: `${itemId}:1`, providerMetadata: { openai: { itemId } } },+ { type: 'reasoning-delta', id: `${itemId}:1`, text: 'second summary' },+ {+ type: 'reasoning-end',+ id: `${itemId}:1`,+ providerMetadata: { openai: { itemId, reasoningEncryptedContent: 'enc-sig' } },+ },+ makeFinishStep('stop'),+ ]),+ chunkMeta: CHUNK_META,+ }),+ );++ expect(final.output.thinking_blocks).toEqual([+ { type: 'thinking', thinking: 'first summary\n\nsecond summary', signature: 'enc-sig' },+ ]);+ });++ // An item can open a summary part it never writes to, which must not push the reasoning that+ // follows behind a blank line.+ it('does not indent the text of an item whose first part is empty', async () => {+ const itemId = 'rs_abc';+ const { final } = await drainStream(+ mapStreamToChunks({+ stream: makeStream([+ { type: 'reasoning-start', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },+ { type: 'reasoning-end', id: `${itemId}:0`, providerMetadata: { openai: { itemId } } },+ { type: 'reasoning-start', id: `${itemId}:1`, providerMetadata: { openai: { itemId } } },+ { type: 'reasoning-delta', id: `${itemId}:1`, text: 'the only summary' },+ {+ type: 'reasoning-end',+ id: `${itemId}:1`,+ providerMetadata: { openai: { itemId, reasoningEncryptedContent: 'enc-item' } },+ },+ makeFinishStep('stop'),+ ]),+ chunkMeta: CHUNK_META,+ }),+ );++ expect(final.output.thinking_blocks).toEqual([+ { type: 'thinking', thinking: 'the only summary', signature: 'enc-item' },+ ]);+ });++ it('keeps separate reasoning items in separate blocks, each with its own token', async () => {+ const { final } = await drainStream(+ mapStreamToChunks({+ stream: makeStream([+ { type: 'reasoning-start', id: 'rs_1:0', providerMetadata: { openai: { itemId: 'rs_1' } } },+ { type: 'reasoning-delta', id: 'rs_1:0', text: 'before the tool call' },+ {+ type: 'reasoning-end',+ id: 'rs_1:0',+ providerMetadata: { openai: { itemId: 'rs_1', reasoningEncryptedContent: 'enc-1' } },+ },+ { type: 'reasoning-start', id: 'rs_2:0', providerMetadata: { openai: { itemId: 'rs_2' } } },+ { type: 'reasoning-delta', id: 'rs_2:0', text: 'after the tool call' },+ {+ type: 'reasoning-end',+ id: 'rs_2:0',+ providerMetadata: { openai: { itemId: 'rs_2', reasoningEncryptedContent: 'enc-2' } },+ },+ makeFinishStep('stop'),+ ]),+ chunkMeta: CHUNK_META,+ }),+ );++ expect(final.output.thinking_blocks).toEqual([+ { type: 'thinking', thinking: 'before the tool call', signature: 'enc-1' },+ { type: 'thinking', thinking: 'after the tool call', signature: 'enc-2' },+ ]);+ });+// Anthropic's real wire shape: signature_delta surfaces as a text-less reasoning-delta, not on// reasoning-end. Missing it leaves the block unsigned, and Anthropic drops unsigned thinking// blocks when they are replayed, silently losing the reasoning chain.