Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: preserve tool result identity in WebGPU VL history
Barack Sokullu93893eadd5c8
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.jsindex 98736b8fd..008753f78 100644--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@ -1089,7 +1089,7 @@ function imageUrlFromBlock(block) {return '';}-function prepareMultimodalMessages(messages) {+export function prepareMultimodalMessages(messages) {const imageUrls = [];const prepared = [];for (const message of Array.isArray(messages) ? messages : []) {@@ -1122,6 +1122,8 @@ function prepareMultimodalMessages(messages) {prepared.push({role,content: blocks,+ ...(typeof message.tool_call_id === 'string' ? { tool_call_id: message.tool_call_id } : {}),+ ...(typeof message.name === 'string' ? { name: message.name } : {}),...(Array.isArray(message.tool_calls)? { tool_calls: message.tool_calls.map(normalizeTextToolCall) }: {}),diff --git a/test/run.js b/test/run.jsindex 68f5c4aeb..f7bfdd595 100644--- a/test/run.js+++ b/test/run.js@@ -64284,7 +64284,7 @@ test('WebGPU worker replays text tool history and applies model-specific generat},};const workerUrl = `${pathToFileURL(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js')).href}?tool-history-test`;- const { prepareTextMessages, splitThinking, tokenizerSupportsTools } = await import(workerUrl);+ const { prepareTextMessages, prepareMultimodalMessages, splitThinking, tokenizerSupportsTools } = await import(workerUrl);const messages = [{role: 'assistant',@@ -64306,6 +64306,30 @@ test('WebGPU worker replays text tool history and applies model-specific generatassert.equal(prepared[0].content, '');assert.equal(prepared[1].content, '{"success":true}');assert.equal(messages[0].tool_calls[0].function.arguments, '{"ref_id":"ref_7","force":true}', 'normalization must not mutate persisted history');+ const multimodalHistory = [+ { role: 'user', content: [+ { type: 'text', text: 'Click the target in this screenshot.' },+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AA==' } },+ ] },+ messages[0],+ { ...messages[1], name: 'click_ax' },+ ];+ const originalHistory = structuredClone(multimodalHistory);+ const multimodal = prepareMultimodalMessages(multimodalHistory);+ assert.deepEqual(multimodal.imageUrls, ['data:image/png;base64,AA==']);+ assert.deepEqual(multimodal.messages[0].content, [+ { type: 'image' },+ { type: 'text', text: 'Click the target in this screenshot.' },+ ]);+ assert.equal(multimodal.messages[1].tool_calls[0].id, 'call_1');+ assert.deepEqual(multimodal.messages[1].tool_calls[0].function.arguments, { ref_id: 'ref_7', force: true });+ assert.deepEqual(multimodal.messages[2], {+ role: 'tool',+ content: [{ type: 'text', text: '{"success":true}' }],+ tool_call_id: 'call_1',+ name: 'click_ax',+ }, 'VL templates must receive the tool result paired with its assistant call');+ assert.deepEqual(multimodalHistory, originalHistory, 'VL normalization must not mutate persisted history');assert.deepEqual(splitThinking('<think>private trace</think>Visible answer'), {content: 'Visible answer',reasoningContent: 'private trace',