Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(agent): support CJK word segmentation and expose recipient clarify purposes
Emre Sokullu5a74e78b208e
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 2c7660d85..c82fb3510 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -17868,20 +17868,51 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d};}- _isRecipientClarification(context, pendingType = 'message_recipient') {+ _isRecipientClarification(context, pendingType = 'message_recipient', observedCandidates = []) {if (!context || typeof context !== 'object') return false;const purpose = String(context.purpose || '').trim();if (pendingType === 'recipient_change') {if (purpose === 'recipient_change') return true;- const text = [context.question, context.reason].filter(Boolean).join(' ').toLowerCase();- return /(change|switch|different|instead|update|replace).*(recipient|conversation|contact|send to|person|address)/i.test(text)- || /(recipient|conversation|contact).*(change|switch|different)/i.test(text)- || /(send|deliver).*(to .* instead|to someone else)/i.test(text)- || /(send|reply).*(instead of)/i.test(text);+ if (purpose === 'research_escalation') return false;+ const text = [+ context.question,+ context.reason,+ ...(Array.isArray(context.options) ? context.options : []),+ ].filter(Boolean).join(' ').toLowerCase();+ const changeWord = /(change|switch|different|instead|update|replace|another|new|更改|更换|切换|替换|换成)/i.test(text);+ const targetWord = /(recipient|conversation|contact|send to|person|address|target|someone else|收件人|联系人|发送)/i.test(text);+ return changeWord && targetWord;}if (purpose === 'message_recipient' || purpose === 'recipient_change') return true;- const text = [context.question, context.reason].filter(Boolean).join(' ').toLowerCase();- return /(recipient|who (is|to|should|are)|send (the |this )?(message|email|draft) to|message (to|for)|deliver to|email to|draft to|authorize.*conversation)/i.test(text);+ if (purpose === 'research_escalation') return false;++ // Check if the clarify question, reason, or options explicitly mention any observed candidate identity+ if (Array.isArray(observedCandidates) && observedCandidates.length > 0) {+ const allCandidateText = [+ context.question,+ context.reason,+ ...(Array.isArray(context.options) ? context.options : []),+ ].filter(Boolean).join(' ');+ for (const candidate of observedCandidates) {+ const id = typeof candidate === 'string' ? candidate : (candidate?.identity ?? candidate?.recipient);+ if (id && answerNamesIdentity(allCandidateText, id)) {+ return true;+ }+ }+ }++ const text = [+ context.question,+ context.reason,+ ...(Array.isArray(context.options) ? context.options : []),+ ].filter(Boolean).join(' ').toLowerCase();++ if (/recipient|收件人/i.test(text)) return true;++ const hasRecipientTerm = /(contact|person|address|who|whom|which contact|which person|which recipient|which user|which address|to whom|for whom|someone|联系人|哪位|谁)/i.test(text);+ const hasMessagingTerm = /(send|receive|deliver|message|email|mail|draft|reply|chat|conversation|发送|邮件|信息|消息|寄|回复)/i.test(text);++ return hasRecipientTerm && hasMessagingTerm;}/**@@ -17922,10 +17953,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!pending || !pendingType || observed.length === 0) return false;if (!answer || source === 'timeout' || source === 'auto') return false;if (pendingType === 'recipient_change') {- if (!clarifyContext || !this._isRecipientClarification(clarifyContext, 'recipient_change')) {+ if (!clarifyContext || !this._isRecipientClarification(clarifyContext, 'recipient_change', observed)) {return false;}- } else if (clarifyContext && !this._isRecipientClarification(clarifyContext, 'message_recipient')) {+ } else if (clarifyContext && !this._isRecipientClarification(clarifyContext, 'message_recipient', observed)) {return false;}const normalizedAnswer = normalizeRecipientIdentity(answer);diff --git a/src/chrome/src/agent/message-recipient-guard.js b/src/chrome/src/agent/message-recipient-guard.jsindex d72ae26a6..1bd051726 100644--- a/src/chrome/src/agent/message-recipient-guard.js+++ b/src/chrome/src/agent/message-recipient-guard.js@@ -58,9 +58,40 @@ export function recipientMatchesObservedIdentity(recipient, observedIdentity) {const IDENTITY_WORD_CHAR = /[\p{L}\p{N}]/u;+let cachedWordSegmenter = null;++function getWordSegmenter() {+ if (cachedWordSegmenter) return cachedWordSegmenter;+ if (typeof Intl !== 'undefined' && typeof Intl.Segmenter === 'function') {+ try {+ cachedWordSegmenter = new Intl.Segmenter(undefined, { granularity: 'word' });+ } catch {+ cachedWordSegmenter = null;+ }+ }+ return cachedWordSegmenter;+}++export function getWordBoundaries(text) {+ const boundaries = new Set([0, text.length]);+ const segmenter = getWordSegmenter();+ if (segmenter) {+ try {+ for (const seg of segmenter.segment(text)) {+ boundaries.add(seg.index);+ boundaries.add(seg.index + seg.segment.length);+ }+ } catch {+ // fallback+ }+ }+ return boundaries;+}+/*** Return all start/end spans where normalizedIdentity occurs in normalizedAnswer- * at word boundaries, or as an exact full-string match for short identities (< 3 chars).+ * at word boundaries (including script-aware segmentation for unsegmented scripts),+ * or as an exact full-string match for short identities (< 3 chars).*/export function findCandidateAnswerSpans(normalizedAnswer, normalizedIdentity) {const answer = normalizeRecipientIdentity(normalizedAnswer);@@ -69,12 +100,17 @@ export function findCandidateAnswerSpans(normalizedAnswer, normalizedIdentity) {if (identity.length < 3) {return answer === identity ? [{ start: 0, end: answer.length }] : [];}+ const boundaries = getWordBoundaries(answer);const spans = [];for (let at = answer.indexOf(identity); at >= 0; at = answer.indexOf(identity, at + 1)) {+ const end = at + identity.length;const before = at > 0 ? answer[at - 1] : '';- const after = answer[at + identity.length] || '';- if (!IDENTITY_WORD_CHAR.test(before) && !IDENTITY_WORD_CHAR.test(after)) {- spans.push({ start: at, end: at + identity.length });+ const after = answer[end] || '';+ const charBoundary = (!before || !IDENTITY_WORD_CHAR.test(before))+ && (!after || !IDENTITY_WORD_CHAR.test(after));+ const segmentBoundary = boundaries.has(at) && boundaries.has(end);+ if (charBoundary || segmentBoundary) {+ spans.push({ start: at, end });}}return spans;diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.jsindex e86fe8e57..c9dbbbcd6 100644--- a/src/chrome/src/agent/tools.js+++ b/src/chrome/src/agent/tools.js@@ -810,8 +810,8 @@ export const AGENT_TOOLS = [},purpose: {type: 'string',- enum: ['research_escalation'],- description: 'Set to research_escalation only when asking to delegate a read-only research subtask.',+ enum: ['research_escalation', 'message_recipient', 'recipient_change'],+ description: 'Optional purpose of the clarification. Use message_recipient to ask the user to confirm or specify the message/email recipient, recipient_change to switch an already-authorized recipient, or research_escalation to delegate a read-only research subtask.',},research_request: {type: 'string',@@ -1595,7 +1595,11 @@ function askResearchConsentTool(tool) {...properties.reason,description: 'Optional one-sentence explanation of why this unusually complex read-only research subtask would benefit from ChatGPT.',},- purpose: properties.purpose,+ purpose: {+ ...properties.purpose,+ enum: ['research_escalation'],+ description: 'Must be set to research_escalation.',+ },research_request: properties.research_request,approve_option: properties.approve_option,},diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex e5d694e52..df9971a55 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -15621,20 +15621,51 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d};}- _isRecipientClarification(context, pendingType = 'message_recipient') {+ _isRecipientClarification(context, pendingType = 'message_recipient', observedCandidates = []) {if (!context || typeof context !== 'object') return false;const purpose = String(context.purpose || '').trim();if (pendingType === 'recipient_change') {if (purpose === 'recipient_change') return true;- const text = [context.question, context.reason].filter(Boolean).join(' ').toLowerCase();- return /(change|switch|different|instead|update|replace).*(recipient|conversation|contact|send to|person|address)/i.test(text)- || /(recipient|conversation|contact).*(change|switch|different)/i.test(text)- || /(send|deliver).*(to .* instead|to someone else)/i.test(text)- || /(send|reply).*(instead of)/i.test(text);+ if (purpose === 'research_escalation') return false;+ const text = [+ context.question,+ context.reason,+ ...(Array.isArray(context.options) ? context.options : []),+ ].filter(Boolean).join(' ').toLowerCase();+ const changeWord = /(change|switch|different|instead|update|replace|another|new|更改|更换|切换|替换|换成)/i.test(text);+ const targetWord = /(recipient|conversation|contact|send to|person|address|target|someone else|收件人|联系人|发送)/i.test(text);+ return changeWord && targetWord;}if (purpose === 'message_recipient' || purpose === 'recipient_change') return true;- const text = [context.question, context.reason].filter(Boolean).join(' ').toLowerCase();- return /(recipient|who (is|to|should|are)|send (the |this )?(message|email|draft) to|message (to|for)|deliver to|email to|draft to|authorize.*conversation)/i.test(text);+ if (purpose === 'research_escalation') return false;++ // Check if the clarify question, reason, or options explicitly mention any observed candidate identity+ if (Array.isArray(observedCandidates) && observedCandidates.length > 0) {+ const allCandidateText = [+ context.question,+ context.reason,+ ...(Array.isArray(context.options) ? context.options : []),+ ].filter(Boolean).join(' ');+ for (const candidate of observedCandidates) {+ const id = typeof candidate === 'string' ? candidate : (candidate?.identity ?? candidate?.recipient);+ if (id && answerNamesIdentity(allCandidateText, id)) {+ return true;+ }+ }+ }++ const text = [+ context.question,+ context.reason,+ ...(Array.isArray(context.options) ? context.options : []),+ ].filter(Boolean).join(' ').toLowerCase();++ if (/recipient|收件人/i.test(text)) return true;++ const hasRecipientTerm = /(contact|person|address|who|whom|which contact|which person|which recipient|which user|which address|to whom|for whom|someone|联系人|哪位|谁)/i.test(text);+ const hasMessagingTerm = /(send|receive|deliver|message|email|mail|draft|reply|chat|conversation|发送|邮件|信息|消息|寄|回复)/i.test(text);++ return hasRecipientTerm && hasMessagingTerm;}/**@@ -15675,10 +15706,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!pending || !pendingType || observed.length === 0) return false;if (!answer || source === 'timeout' || source === 'auto') return false;if (pendingType === 'recipient_change') {- if (!clarifyContext || !this._isRecipientClarification(clarifyContext, 'recipient_change')) {+ if (!clarifyContext || !this._isRecipientClarification(clarifyContext, 'recipient_change', observed)) {return false;}- } else if (clarifyContext && !this._isRecipientClarification(clarifyContext, 'message_recipient')) {+ } else if (clarifyContext && !this._isRecipientClarification(clarifyContext, 'message_recipient', observed)) {return false;}const normalizedAnswer = normalizeRecipientIdentity(answer);diff --git a/src/firefox/src/agent/message-recipient-guard.js b/src/firefox/src/agent/message-recipient-guard.jsindex d72ae26a6..1bd051726 100644--- a/src/firefox/src/agent/message-recipient-guard.js+++ b/src/firefox/src/agent/message-recipient-guard.js@@ -58,9 +58,40 @@ export function recipientMatchesObservedIdentity(recipient, observedIdentity) {const IDENTITY_WORD_CHAR = /[\p{L}\p{N}]/u;+let cachedWordSegmenter = null;++function getWordSegmenter() {+ if (cachedWordSegmenter) return cachedWordSegmenter;+ if (typeof Intl !== 'undefined' && typeof Intl.Segmenter === 'function') {+ try {+ cachedWordSegmenter = new Intl.Segmenter(undefined, { granularity: 'word' });+ } catch {+ cachedWordSegmenter = null;+ }+ }+ return cachedWordSegmenter;+}++export function getWordBoundaries(text) {+ const boundaries = new Set([0, text.length]);+ const segmenter = getWordSegmenter();+ if (segmenter) {+ try {+ for (const seg of segmenter.segment(text)) {+ boundaries.add(seg.index);+ boundaries.add(seg.index + seg.segment.length);+ }+ } catch {+ // fallback+ }+ }+ return boundaries;+}+/*** Return all start/end spans where normalizedIdentity occurs in normalizedAnswer- * at word boundaries, or as an exact full-string match for short identities (< 3 chars).+ * at word boundaries (including script-aware segmentation for unsegmented scripts),+ * or as an exact full-string match for short identities (< 3 chars).*/export function findCandidateAnswerSpans(normalizedAnswer, normalizedIdentity) {const answer = normalizeRecipientIdentity(normalizedAnswer);@@ -69,12 +100,17 @@ export function findCandidateAnswerSpans(normalizedAnswer, normalizedIdentity) {if (identity.length < 3) {return answer === identity ? [{ start: 0, end: answer.length }] : [];}+ const boundaries = getWordBoundaries(answer);const spans = [];for (let at = answer.indexOf(identity); at >= 0; at = answer.indexOf(identity, at + 1)) {+ const end = at + identity.length;const before = at > 0 ? answer[at - 1] : '';- const after = answer[at + identity.length] || '';- if (!IDENTITY_WORD_CHAR.test(before) && !IDENTITY_WORD_CHAR.test(after)) {- spans.push({ start: at, end: at + identity.length });+ const after = answer[end] || '';+ const charBoundary = (!before || !IDENTITY_WORD_CHAR.test(before))+ && (!after || !IDENTITY_WORD_CHAR.test(after));+ const segmentBoundary = boundaries.has(at) && boundaries.has(end);+ if (charBoundary || segmentBoundary) {+ spans.push({ start: at, end });}}return spans;diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.jsindex 4aa4b4ea9..26289ce3c 100644--- a/src/firefox/src/agent/tools.js+++ b/src/firefox/src/agent/tools.js@@ -672,8 +672,8 @@ export const AGENT_TOOLS = [},purpose: {type: 'string',- enum: ['research_escalation'],- description: 'Set to research_escalation only when asking to delegate a read-only research subtask.',+ enum: ['research_escalation', 'message_recipient', 'recipient_change'],+ description: 'Optional purpose of the clarification. Use message_recipient to ask the user to confirm or specify the message/email recipient, recipient_change to switch an already-authorized recipient, or research_escalation to delegate a read-only research subtask.',},research_request: {type: 'string',@@ -1442,7 +1442,11 @@ function askResearchConsentTool(tool) {...properties.reason,description: 'Optional one-sentence explanation of why this unusually complex read-only research subtask would benefit from ChatGPT.',},- purpose: properties.purpose,+ purpose: {+ ...properties.purpose,+ enum: ['research_escalation'],+ description: 'Must be set to research_escalation.',+ },research_request: properties.research_request,approve_option: properties.approve_option,},diff --git a/test/run.js b/test/run.jsindex 132f2e557..ead5a0401 100644--- a/test/run.js+++ b/test/run.js@@ -2715,6 +2715,11 @@ test('research escalation is opt-in, tier-complete when enabled, and absent othefor (const field of researchOnlyClarifyFields) {assert.ok(actClarify.function.parameters.properties[field], `${label}: enabled Act clarification lost ${field}`);}+ assert.deepEqual(+ actClarify.function.parameters.properties.purpose.enum,+ ['research_escalation', 'message_recipient', 'recipient_change'],+ `${label}: clarify purpose did not expose recipient purposes`,+ );assert.deepEqual(actClarify.function.parameters.required, ['question']);assert.doesNotMatch(actClarify.function.description, /not a generic clarification tool/i);@@ -4646,6 +4651,8 @@ test('direct-message recipient guard uses structured intent and exact active ideassert.equal(helper.answerNamesIdentity('Lisa', 'Li'), false);assert.equal(helper.answerNamesIdentity('王', '王'), true);assert.equal(helper.answerNamesIdentity('王小明', '王'), false);+ assert.equal(helper.answerNamesIdentity('请发送给王小明', '王小明'), true, 'natural Chinese sentence must match CJK recipient');+ assert.equal(helper.answerNamesIdentity('请发送给王小明', '王'), false, 'sub-fragment of CJK name must not match short identity');assert.equal(helper.answerNamesAllObservedRecipients('Send to Ann Smith',[{ identity: 'Ann' }, { identity: 'Ann Smith' }],@@ -5248,6 +5255,26 @@ test('direct-message recipient guard uses structured intent and exact active ide`${label}: valid recipient clarification context was rejected`,);+ // Natural recipient question without explicit purpose (e.g. "Which contact should receive this message?").+ agent._planExecutionGuards.set(tabId, clarifyGuardState([{ identity: '[email protected]', role: 'to' }]));+ assert.equal(+ agent._bindClarifiedMessageRecipient(tabId, '[email protected]', 'user', {+ question: 'Which contact should receive this message?',+ }),+ true,+ `${label}: natural recipient question ("Which contact should receive this message?") was rejected`,+ );++ // Natural-language Chinese answer for CJK recipient.+ agent._planExecutionGuards.set(tabId, clarifyGuardState([{ identity: '王小明', role: 'to' }]));+ assert.equal(+ agent._bindClarifiedMessageRecipient(tabId, '请发送给王小明', 'user', {+ question: '请确认要发送给哪位联系人?',+ }),+ true,+ `${label}: natural Chinese clarification question/answer was rejected`,+ );+// Recipient-change clarification check: when a target was already set, an explicit// recipient_change context is required.const recipientChangeGuardState = (candidates) => ({