Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(agent): bind published URLs to occurrence order and support multilingual task body extraction
Emre Sokulluc4535677a009
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex e7524da60..68e450ffa 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -2305,16 +2305,23 @@ export class Agent extends LoopDetector {return renderedRaw ? rawUrl : trimmed;});if (requestedUrls.length < 1) return false;- let comparableExpected = expectedBody;- let comparableObserved = observedBody;const consumedLinks = new Set();const priorLinkByRequestedUrl = new Map();- const replaceFirst = (text, value, replacement) => {- const offset = text.indexOf(value);- return offset < 0- ? ''- : text.slice(0, offset) + replacement + text.slice(offset + value.length);+ const linkUsageCount = new Map();+ const expectedOccurrencesByUrl = new Map();+ const expectedReplacements = [];+ const observedReplacements = [];++ const findOccurrences = (text, searchStr) => {+ const res = [];+ let idx = 0;+ while ((idx = text.indexOf(searchStr, idx)) !== -1) {+ res.push({ start: idx, end: idx + searchStr.length });+ idx += searchStr.length;+ }+ return res;};+for (const [index, requested] of requestedUrls.entries()) {let linkIndex = links.findIndex((candidate, candidateIndex) => (!consumedLinks.has(candidateIndex)@@ -2327,14 +2334,53 @@ export class Agent extends LoopDetector {if (!priorLinkByRequestedUrl.has(requested)) priorLinkByRequestedUrl.set(requested, linkIndex);const displayed = ['text', 'title', 'ariaLabel', 'expandedUrl', 'href'].map(field => this._workflowMetadataValue(link[field]))- .filter(value => value && comparableObserved.includes(value))+ .filter(value => value && observedBody.includes(value)).find(value => this._workflowSocialDisplayedUrlMatchesRequested(value, requested));if (!displayed) return false;++ const timesUsed = linkUsageCount.get(linkIndex) || 0;+ linkUsageCount.set(linkIndex, timesUsed + 1);++ let priorDisplayOccurrences = 0;+ for (let i = 0; i < linkIndex; i++) {+ const hasMatchingDisplay = ['text', 'title', 'ariaLabel', 'expandedUrl', 'href']+ .map(field => this._workflowMetadataValue(links[i]?.[field]))+ .some(val => val === displayed);+ if (hasMatchingDisplay) {+ const usage = linkUsageCount.get(i) || 0;+ priorDisplayOccurrences += Math.max(1, usage);+ }+ }+ const occurrenceIndex = priorDisplayOccurrences + timesUsed;++ const observedOccurrences = findOccurrences(observedBody, displayed);+ if (occurrenceIndex >= observedOccurrences.length) return false;+ const observedRange = observedOccurrences[occurrenceIndex];++ const expTimesUsed = expectedOccurrencesByUrl.get(requested) || 0;+ expectedOccurrencesByUrl.set(requested, expTimesUsed + 1);+ const expOccurrences = findOccurrences(expectedBody, requested);+ if (expTimesUsed >= expOccurrences.length) return false;+ const expectedRange = expOccurrences[expTimesUsed];+const marker = `__WEBBRAIN_PUBLISHED_URL_${index}__`;- comparableExpected = replaceFirst(comparableExpected, requested, marker);- comparableObserved = replaceFirst(comparableObserved, displayed, marker);- if (!comparableExpected || !comparableObserved) return false;+ expectedReplacements.push({ ...expectedRange, marker });+ observedReplacements.push({ ...observedRange, marker });}++ const applyReplacements = (text, replacements) => {+ const sorted = [...replacements].sort((a, b) => b.start - a.start);+ let res = text;+ for (const r of sorted) {+ res = res.slice(0, r.start) + r.marker + res.slice(r.end);+ }+ return res;+ };++ const comparableExpected = applyReplacements(expectedBody, expectedReplacements);+ const comparableObserved = applyReplacements(observedBody, observedReplacements);+ if (!comparableExpected || !comparableObserved) return false;+if (hasDedicatedAuthoredText) {return comparableExpected === comparableObserved;}@@ -22160,21 +22206,25 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d.map(t => String(t || '').trim()).filter(Boolean);let bestBody = '';+ const publishVerbPattern = `(?:${SOCIAL_PUBLISH_VERBS.source}|message|update)`;+ const quotedPattern = new RegExp(`${publishVerbPattern}[\\s\\S]*?(?:“([\\s\\S]+?)”|「([\\s\\S]+?)」|『([\\s\\S]+?)』|«([\\s\\S]+?)»|"([\\s\\S]+?)")`, 'iu');+ const singleQuotePattern = new RegExp(`${publishVerbPattern}[\\s\\S]*?(?:(?<!\\p{L})'([\\s\\S]+?)'(?!\\p{L}))`, 'iu');+ const colonPattern = new RegExp(`${publishVerbPattern}[\\s\\S]*?(?<!https?|ftp|sftp)[::](?!//)\\s*([\\s\\S]+)$`, 'iu');for (const text of candidates) {let body = '';// 1. Quoted body: Post "..." on X, Tweet "...", Publish "...", etc.- const quotedMatch = text.match(/(?:publish(?:ed|ing|es)?|post(?:ed|ing|s)?|tweet(?:ed|ing|s)?|skeet(?:ed|ing|s)?|share|sharing|update)[\s\S]*?(?:“([\s\S]+?)”|「([\s\S]+?)」|『([\s\S]+?)』|«([\s\S]+?)»|"([\s\S]+?)")/i);+ const quotedMatch = text.match(quotedPattern);const matchedContent = quotedMatch ? (quotedMatch[1] || quotedMatch[2] || quotedMatch[3] || quotedMatch[4] || quotedMatch[5]) : null;if (matchedContent && matchedContent.trim()) {body = matchedContent.trim();} else {// Single-quoted body (not contractions)- const singleQuoteMatch = text.match(/(?:publish(?:ed|ing|es)?|post(?:ed|ing|s)?|tweet(?:ed|ing|s)?|skeet(?:ed|ing|s)?|share|sharing|update)[\s\S]*?(?:(?<!\p{L})'([\s\S]+?)'(?!\p{L}))/iu);+ const singleQuoteMatch = text.match(singleQuotePattern);if (singleQuoteMatch && singleQuoteMatch[1]?.trim()) {body = singleQuoteMatch[1].trim();} else {// 2. Colon-introduced body: "Publish this exact post on X:\s*<body text>", "Tweet:\s*<body text>", "Post the following on X:\s*<body text>"- const colonMatch = text.match(/(?:publish(?:ing|es)?|post(?:ing|s)?|tweet(?:ing|s)?|message|update)[\s\S]*?(?<!https?|ftp|sftp):(?!\/\/)\s*([\s\S]+)$/i);+ const colonMatch = text.match(colonPattern);if (colonMatch && colonMatch[1]?.trim()) {let candidateBody = colonMatch[1].trim();const innerQuote = candidateBody.match(/^(?:“([\s\S]+)”|「([\s\S]+)」|『([\s\S]+)』|«([\s\S]+)»|"([\s\S]+)"|'([\s\S]+)')$/);diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex d8251002a..ea9c52b99 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -2019,16 +2019,23 @@ export class Agent extends LoopDetector {return renderedRaw ? rawUrl : trimmed;});if (requestedUrls.length < 1) return false;- let comparableExpected = expectedBody;- let comparableObserved = observedBody;const consumedLinks = new Set();const priorLinkByRequestedUrl = new Map();- const replaceFirst = (text, value, replacement) => {- const offset = text.indexOf(value);- return offset < 0- ? ''- : text.slice(0, offset) + replacement + text.slice(offset + value.length);+ const linkUsageCount = new Map();+ const expectedOccurrencesByUrl = new Map();+ const expectedReplacements = [];+ const observedReplacements = [];++ const findOccurrences = (text, searchStr) => {+ const res = [];+ let idx = 0;+ while ((idx = text.indexOf(searchStr, idx)) !== -1) {+ res.push({ start: idx, end: idx + searchStr.length });+ idx += searchStr.length;+ }+ return res;};+for (const [index, requested] of requestedUrls.entries()) {let linkIndex = links.findIndex((candidate, candidateIndex) => (!consumedLinks.has(candidateIndex)@@ -2041,14 +2048,53 @@ export class Agent extends LoopDetector {if (!priorLinkByRequestedUrl.has(requested)) priorLinkByRequestedUrl.set(requested, linkIndex);const displayed = ['text', 'title', 'ariaLabel', 'expandedUrl', 'href'].map(field => this._workflowMetadataValue(link[field]))- .filter(value => value && comparableObserved.includes(value))+ .filter(value => value && observedBody.includes(value)).find(value => this._workflowSocialDisplayedUrlMatchesRequested(value, requested));if (!displayed) return false;++ const timesUsed = linkUsageCount.get(linkIndex) || 0;+ linkUsageCount.set(linkIndex, timesUsed + 1);++ let priorDisplayOccurrences = 0;+ for (let i = 0; i < linkIndex; i++) {+ const hasMatchingDisplay = ['text', 'title', 'ariaLabel', 'expandedUrl', 'href']+ .map(field => this._workflowMetadataValue(links[i]?.[field]))+ .some(val => val === displayed);+ if (hasMatchingDisplay) {+ const usage = linkUsageCount.get(i) || 0;+ priorDisplayOccurrences += Math.max(1, usage);+ }+ }+ const occurrenceIndex = priorDisplayOccurrences + timesUsed;++ const observedOccurrences = findOccurrences(observedBody, displayed);+ if (occurrenceIndex >= observedOccurrences.length) return false;+ const observedRange = observedOccurrences[occurrenceIndex];++ const expTimesUsed = expectedOccurrencesByUrl.get(requested) || 0;+ expectedOccurrencesByUrl.set(requested, expTimesUsed + 1);+ const expOccurrences = findOccurrences(expectedBody, requested);+ if (expTimesUsed >= expOccurrences.length) return false;+ const expectedRange = expOccurrences[expTimesUsed];+const marker = `__WEBBRAIN_PUBLISHED_URL_${index}__`;- comparableExpected = replaceFirst(comparableExpected, requested, marker);- comparableObserved = replaceFirst(comparableObserved, displayed, marker);- if (!comparableExpected || !comparableObserved) return false;+ expectedReplacements.push({ ...expectedRange, marker });+ observedReplacements.push({ ...observedRange, marker });}++ const applyReplacements = (text, replacements) => {+ const sorted = [...replacements].sort((a, b) => b.start - a.start);+ let res = text;+ for (const r of sorted) {+ res = res.slice(0, r.start) + r.marker + res.slice(r.end);+ }+ return res;+ };++ const comparableExpected = applyReplacements(expectedBody, expectedReplacements);+ const comparableObserved = applyReplacements(observedBody, observedReplacements);+ if (!comparableExpected || !comparableObserved) return false;+if (hasDedicatedAuthoredText) {return comparableExpected === comparableObserved;}@@ -19763,21 +19809,25 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d.map(t => String(t || '').trim()).filter(Boolean);let bestBody = '';+ const publishVerbPattern = `(?:${SOCIAL_PUBLISH_VERBS.source}|message|update)`;+ const quotedPattern = new RegExp(`${publishVerbPattern}[\\s\\S]*?(?:“([\\s\\S]+?)”|「([\\s\\S]+?)」|『([\\s\\S]+?)』|«([\\s\\S]+?)»|"([\\s\\S]+?)")`, 'iu');+ const singleQuotePattern = new RegExp(`${publishVerbPattern}[\\s\\S]*?(?:(?<!\\p{L})'([\\s\\S]+?)'(?!\\p{L}))`, 'iu');+ const colonPattern = new RegExp(`${publishVerbPattern}[\\s\\S]*?(?<!https?|ftp|sftp)[::](?!//)\\s*([\\s\\S]+)$`, 'iu');for (const text of candidates) {let body = '';// 1. Quoted body: Post "..." on X, Tweet "...", Publish "...", etc.- const quotedMatch = text.match(/(?:publish(?:ed|ing|es)?|post(?:ed|ing|s)?|tweet(?:ed|ing|s)?|skeet(?:ed|ing|s)?|share|sharing|update)[\s\S]*?(?:“([\s\S]+?)”|「([\s\S]+?)」|『([\s\S]+?)』|«([\s\S]+?)»|"([\s\S]+?)")/i);+ const quotedMatch = text.match(quotedPattern);const matchedContent = quotedMatch ? (quotedMatch[1] || quotedMatch[2] || quotedMatch[3] || quotedMatch[4] || quotedMatch[5]) : null;if (matchedContent && matchedContent.trim()) {body = matchedContent.trim();} else {// Single-quoted body (not contractions)- const singleQuoteMatch = text.match(/(?:publish(?:ed|ing|es)?|post(?:ed|ing|s)?|tweet(?:ed|ing|s)?|skeet(?:ed|ing|s)?|share|sharing|update)[\s\S]*?(?:(?<!\p{L})'([\s\S]+?)'(?!\p{L}))/iu);+ const singleQuoteMatch = text.match(singleQuotePattern);if (singleQuoteMatch && singleQuoteMatch[1]?.trim()) {body = singleQuoteMatch[1].trim();} else {// 2. Colon-introduced body: "Publish this exact post on X:\s*<body text>", "Tweet:\s*<body text>", "Post the following on X:\s*<body text>"- const colonMatch = text.match(/(?:publish(?:ing|es)?|post(?:ing|s)?|tweet(?:ing|s)?|message|update)[\s\S]*?(?<!https?|ftp|sftp):(?!\/\/)\s*([\s\S]+)$/i);+ const colonMatch = text.match(colonPattern);if (colonMatch && colonMatch[1]?.trim()) {let candidateBody = colonMatch[1].trim();const innerQuote = candidateBody.match(/^(?:“([\s\S]+)”|「([\s\S]+)」|『([\s\S]+)』|«([\s\S]+)»|"([\s\S]+)"|'([\s\S]+)')$/);diff --git a/test/run.js b/test/run.jsindex 02082cc6e..2d41ba7b4 100644--- a/test/run.js+++ b/test/run.js@@ -86432,6 +86432,19 @@ test('X and Bluesky same-route publication accepts only one new permalink with t],},), true, AgentClass.name + ': repeated shortened URL labels were not consumed positionally');+ assert.equal(agent._workflowSocialPublishedBodyObserved(+ {+ field: 'body',+ value: `First: ${firstLongUrl}\nSecond: ${secondLongUrl}`,+ },+ {+ text: `First: ${repeatedDisplayUrl}\nSecond: ${repeatedDisplayUrl}`,+ links: [+ { href: 'https://t.co/second', text: repeatedDisplayUrl, expandedUrl: secondLongUrl },+ { href: 'https://t.co/first', text: repeatedDisplayUrl, expandedUrl: firstLongUrl },+ ],+ },+ ), false, AgentClass.name + ': swapped link destinations were incorrectly accepted');const guard = agent._startPlanExecutionGuard(tabId, 'act', {requestKind: 'execute',requiresStateChange: true,@@ -87410,6 +87423,52 @@ test('extracting post body skips URL scheme colons', () => {}});+test('extracting post body supports multilingual commands with colons and quotes', () => {+ for (const AgentClass of [AgentCh, AgentFx]) {+ const agent = new AgentClass({});+ assert.equal(+ agent._extractWorkflowTaskBody('Publica en X: Este es un texto largo para publicar'),+ 'Este es un texto largo para publicar',+ AgentClass.name + ': Spanish colon extraction failed',+ );+ assert.equal(+ agent._extractWorkflowTaskBody('Publícalo en Bluesky: «Bonjour tout le monde»'),+ 'Bonjour tout le monde',+ AgentClass.name + ': French colon and guillemets extraction failed',+ );+ assert.equal(+ agent._extractWorkflowTaskBody('Veröffentliche auf X: Dies ist ein langer Text'),+ 'Dies ist ein langer Text',+ AgentClass.name + ': German colon extraction failed',+ );+ assert.equal(+ agent._extractWorkflowTaskBody('Опубликуй в X: Длинный текст поста'),+ 'Длинный текст поста',+ AgentClass.name + ': Russian colon extraction failed',+ );+ assert.equal(+ agent._extractWorkflowTaskBody('Xに投稿:これはテスト投稿です'),+ 'これはテスト投稿です',+ AgentClass.name + ': Japanese full-width colon extraction failed',+ );+ assert.equal(+ agent._extractWorkflowTaskBody('在X发布:这是长文本内容'),+ '这是长文本内容',+ AgentClass.name + ': Chinese full-width colon extraction failed',+ );+ assert.equal(+ agent._extractWorkflowTaskBody('X에 게시: 이것은 게시물 내용입니다'),+ '이것은 게시물 내용입니다',+ AgentClass.name + ': Korean colon extraction failed',+ );+ assert.equal(+ agent._extractWorkflowTaskBody('Publica en X "texto largo aquí"'),+ 'texto largo aquí',+ AgentClass.name + ': Spanish quoted extraction failed',+ );+ }+});+test('classifier fallback leaves metadata requirements incomplete and unresolved', async () => {for (const [index, AgentClass] of [AgentCh, AgentFx].entries()) {const agent = new AgentClass({});