Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Classify complete long follow-ups without losing task context
Emre Sokullu3de7b143d3ba
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex b5f56590b..306282f7f 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -11,7 +11,7 @@ import { completionResultSnapshot, completionResultPixelRect } from './completioimport { captureSemanticNavigationDocument, semanticCommitNavigationRequest, recordSemanticCommitNavigation, semanticCommitIsCurrent, canRecordSemanticCommitDispatch } from './semantic-commit-navigation.js';import { installPageFeedback, beforePageAgentDispatch, capturePageAgentInputCompletion, hasPageAgentDispatchOwner, pageFeedbackPendingResult } from './page-feedback.js';import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';-import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText } from './systemone-evidence.js';+import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText, systemOneStateBytes } from './systemone-evidence.js';import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';import { SEMANTIC_BATCH_LIMITS, SEMANTIC_DECISION_VERSION, acceptSemanticAnswer, acceptSemanticBatchAnswers, boundedSemanticData, controlEffectCommits, semanticBatchChunks, semanticBatchDecisionMessages, semanticDecisionMessages, semanticQuestion, semanticSystemOneBatchQuestions, semanticSystemOneQuestion, WORKFLOW_STATES } from './semantic-decisions.js';import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationContract, resolvePublicationText, publicationProgress, exactPublicationText, publicationMediaMatches, publicationContractMessages, publicationAuditMessages, parsePublicationJson, publicationAuditValidation } from './social-publish-contract.js';@@ -7863,7 +7863,9 @@ export class Agent extends LoopDetector {*/async _semanticDecision(tabId, kind, { request = '', page = {} } = {}, { provider = this._activeProvider(tabId), signal = null, mainModel = true } = {}) {const spec = semanticQuestion(kind);- const trusted = String(request ?? '').slice(0, 6000);+ // Relationship decisions must read the entire latest user message. A+ // prefix-only verdict can mistake a correction or a new task at its end.+ const trusted = kind === 'user_turn_kind' ? String(request ?? '') : String(request ?? '').slice(0, 6000);const data = boundedSemanticData(page);const cacheKey = this._sha256TextSync(JSON.stringify([SEMANTIC_DECISION_VERSION, kind, trusted, data]));this._semanticDecisionCache ??= new Map();@@ -7878,10 +7880,14 @@ export class Agent extends LoopDetector {const context = this.systemOneContext(tabId);try {const settings = await this._jevSettings(tabId, provider);- if (settings && !this._checkAbort(tabId)) {+ const decisionText = redactSystemOneText(JSON.stringify({ request: trusted, page: data }));+ // Keep the existing bounded fast route, but use the full-text main+ // model directly when a relationship question cannot fit in it.+ const decisionFits = kind !== 'user_turn_kind' || systemOneStateBytes(decisionText) <= 12000;+ if (settings && decisionFits && !this._checkAbort(tabId)) {const verdict = await this.evaluateSystemOne(tabId, createSystemOneJudge({ timeoutMs: 2500, maxRetries: 0 }), {config: settings.decisionConfig, apiKey: settings.typesafeApiKey, signal: runSignal,- state: { context: wrapSystemOneData(boundedSystemOneText(redactSystemOneText(JSON.stringify({ request: trusted, page: data })), 12000)) },+ state: { context: wrapSystemOneData(boundedSystemOneText(decisionText, 12000)) },questions: { [kind]: semanticSystemOneQuestion(kind) },}, context);const answer = verdict.answers?.[kind];@@ -7889,6 +7895,8 @@ export class Agent extends LoopDetector {this.recordSystemOneVerdict(tabId, { decision: choice ? 'semantic' : 'fallback', reason: choice ? kind : `${kind}_low_confidence`,model: verdict.model }, context);if (choice) return remember({ choice, evidence: '', source: 'decision_model' });+ } else if (settings && !decisionFits) {+ this.recordSystemOneVerdict(tabId, { decision: 'fallback', reason: `${kind}_context_budget` }, context);}} catch (error) {if (this._isCostAllowanceError(error) || this._checkAbort(tabId)) throw error;@@ -33479,7 +33487,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dasync _userTurnKind(tabId, messages, text, mode, runOptions = {}, sourceBound = false) {if (runOptions?.trustedContinuation === true) return 'continue';const value = typeof text === 'string' ? text.trim() : '';- if (sourceBound || !this._isActionMode(mode) || !value || value.length > 6000+ if (sourceBound || !this._isActionMode(mode) || !value|| this._isScheduledResumeTurn(value) || this._activeTaskBinding(messages || []).index < 0) return 'request';const reference = this._userTurnReference(messages);const verdict = await this._semanticDecision(tabId, 'user_turn_kind', { request: value, page: reference });diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 2c2dca314..699b59edf 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -9,7 +9,7 @@ import { completionResultSnapshot, completionResultPixelRect } from './completioimport { captureSemanticNavigationDocument, semanticCommitNavigationRequest, recordSemanticCommitNavigation, semanticCommitIsCurrent, canRecordSemanticCommitDispatch } from './semantic-commit-navigation.js';import { installPageFeedback, beforePageAgentDispatch, hasPageAgentDispatchOwner, pageFeedbackPendingResult } from './page-feedback.js';import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';-import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText } from './systemone-evidence.js';+import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText, systemOneStateBytes } from './systemone-evidence.js';import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';import { SEMANTIC_BATCH_LIMITS, SEMANTIC_DECISION_VERSION, acceptSemanticAnswer, acceptSemanticBatchAnswers, boundedSemanticData, controlEffectCommits, semanticBatchChunks, semanticBatchDecisionMessages, semanticDecisionMessages, semanticQuestion, semanticSystemOneBatchQuestions, semanticSystemOneQuestion, WORKFLOW_STATES } from './semantic-decisions.js';import { firefoxBidi } from '../bidi/client.js';@@ -7688,7 +7688,9 @@ export class Agent extends LoopDetector {*/async _semanticDecision(tabId, kind, { request = '', page = {} } = {}, { provider = this._activeProvider(tabId), signal = null, mainModel = true } = {}) {const spec = semanticQuestion(kind);- const trusted = String(request ?? '').slice(0, 6000);+ // Relationship decisions must read the entire latest user message. A+ // prefix-only verdict can mistake a correction or a new task at its end.+ const trusted = kind === 'user_turn_kind' ? String(request ?? '') : String(request ?? '').slice(0, 6000);const data = boundedSemanticData(page);const cacheKey = this._sha256TextSync(JSON.stringify([SEMANTIC_DECISION_VERSION, kind, trusted, data]));this._semanticDecisionCache ??= new Map();@@ -7703,10 +7705,14 @@ export class Agent extends LoopDetector {const context = this.systemOneContext(tabId);try {const settings = await this._jevSettings(tabId, provider);- if (settings && !this._checkAbort(tabId)) {+ const decisionText = redactSystemOneText(JSON.stringify({ request: trusted, page: data }));+ // Keep the existing bounded fast route, but use the full-text main+ // model directly when a relationship question cannot fit in it.+ const decisionFits = kind !== 'user_turn_kind' || systemOneStateBytes(decisionText) <= 12000;+ if (settings && decisionFits && !this._checkAbort(tabId)) {const verdict = await this.evaluateSystemOne(tabId, createSystemOneJudge({ timeoutMs: 2500, maxRetries: 0 }), {config: settings.decisionConfig, apiKey: settings.typesafeApiKey, signal: runSignal,- state: { context: wrapSystemOneData(boundedSystemOneText(redactSystemOneText(JSON.stringify({ request: trusted, page: data })), 12000)) },+ state: { context: wrapSystemOneData(boundedSystemOneText(decisionText, 12000)) },questions: { [kind]: semanticSystemOneQuestion(kind) },}, context);const answer = verdict.answers?.[kind];@@ -7714,6 +7720,8 @@ export class Agent extends LoopDetector {this.recordSystemOneVerdict(tabId, { decision: choice ? 'semantic' : 'fallback', reason: choice ? kind : `${kind}_low_confidence`,model: verdict.model }, context);if (choice) return remember({ choice, evidence: '', source: 'decision_model' });+ } else if (settings && !decisionFits) {+ this.recordSystemOneVerdict(tabId, { decision: 'fallback', reason: `${kind}_context_budget` }, context);}} catch (error) {if (this._isCostAllowanceError(error) || this._checkAbort(tabId)) throw error;@@ -30758,7 +30766,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dasync _userTurnKind(tabId, messages, text, mode, runOptions = {}, sourceBound = false) {if (runOptions?.trustedContinuation === true) return 'continue';const value = typeof text === 'string' ? text.trim() : '';- if (sourceBound || !this._isActionMode(mode) || !value || value.length > 6000+ if (sourceBound || !this._isActionMode(mode) || !value|| this._isScheduledResumeTurn(value) || this._activeTaskBinding(messages || []).index < 0) return 'request';const reference = this._userTurnReference(messages);const verdict = await this._semanticDecision(tabId, 'user_turn_kind', { request: value, page: reference });diff --git a/test/follow-up-task-context.mjs b/test/follow-up-task-context.mjsindex 62aa7cdaa..e6866e1c5 100644--- a/test/follow-up-task-context.mjs+++ b/test/follow-up-task-context.mjs@@ -304,6 +304,64 @@ for (const build of ['chrome', 'firefox']) {assert.equal(agent._progressTaskAnchorText(tab), root);});+ for (const [length, character, fast] of [[6001, 'x', true], [14000, 'x', false], [4500, '界', false], [6001, '"', false]]) {+ test(`${build}: ${length}-character ${character} corrections retain the original task after full-text classification`, async () => {+ const { agent, messages, provider } = fixture();+ agent._recordTaskClarification(tab, 'Which bot?', 'Chat with @emrebot', 'option');+ const earlier = 'Use a short greeting, then keep talking.';+ messages.push(user(earlier, 'update'));+ const tail = ' END-OF-LONG-CORRECTION: keep this same task.';+ const correction = character.repeat(length - tail.length) + tail;+ const calls = installModelStub(agent, { provider, decide: (_kind, { request }) => {+ assert.equal(request, correction, 'Fallback must read the entire message, including the final instruction.');+ return { choice: 'update', evidence: tail.trim() };+ } });+ let decisions = 0;+ agent._jevSettings = async () => ({ decisionConfig: { enabled: true, local: true } });+ agent.evaluateSystemOne = async (_tab, _judge, input) => {+ decisions++;+ assert.ok(JSON.stringify(input.state).includes(tail), 'Fast decision must also see the final instruction.');+ return { model: 'owned-decider', answers: { user_turn_kind: { type: 'choice', choice: 'update', confidence: .97,+ probabilities: { continue: .01, update: .97, acknowledge: .01, request: .01 } } } };+ };+ const turn = user(correction);+ await agent._markUserTurnKind(tab, messages, turn, correction, 'act');+ assert.equal(turn.webbrainTurnKind, 'update');+ assert.equal(decisions, fast ? 1 : 0, 'Oversized decision payloads go directly to the full-text LLM fallback.');+ assert.equal(calls.length, fast ? 0 : 1);+ messages.push(turn);+ assert.equal(agent._activeTaskBinding(messages).taskIndex, 1);+ const task = agent._plannerAuthenticTaskText(messages);+ for (const text of [root, earlier, 'Chat with @emrebot']) assert.ok(task.includes(text));+ assert.equal(JSON.parse(task.slice(task.indexOf('{'))).latest_request, correction);+ const checked = JSON.parse(buildPlannerAlignmentMessages(task, chatPlan(), { taskBinding: 'long-related-task' }).at(-1).content);+ assert.equal(checked.current_request, task);+ });+ }++ test(`${build}: long separate requests and cache entries keep their distinguishing suffix`, async () => {+ const { agent, messages, provider } = fixture();+ const prefix = 'x'.repeat(14000);+ const suffixes = [' Keep working on this same dialog.', ' Instead start a separate task: summarize the article.'];+ let decisions = 0;+ agent._jevSettings = async () => ({ decisionConfig: { enabled: true, local: true } });+ agent.evaluateSystemOne = async () => { decisions++; throw new Error('A truncated decision is not permitted'); };+ const calls = installModelStub(agent, { provider, decide: (_kind, { request }) => {+ assert.ok(suffixes.some(suffix => request.endsWith(suffix)));+ return { choice: request.endsWith(suffixes[0]) ? 'update' : 'request', evidence: request.slice(prefix.length).trim() };+ } });+ for (const [index, suffix] of suffixes.entries()) {+ const text = prefix + suffix, turn = user(text);+ await agent._markUserTurnKind(tab, messages, turn, text, 'act');+ assert.equal(turn.webbrainTurnKind, index === 0 ? 'update' : 'request');+ const view = [...messages, turn];+ assert.equal(agent._activeTaskBinding(view).taskIndex, index === 0 ? 1 : messages.length);+ if (index === 1) assert.equal(agent._plannerAuthenticTaskText(view), text);+ }+ assert.equal(calls.length, 2, 'Messages sharing a long prefix must not reuse the other suffix verdict.');+ assert.equal(decisions, 0);+ });+test(`${build}: legacy turns and classification failure preserve the entire latest request`, async () => {const { agent, messages } = fixture();installModelStub(agent, { decide: () => null });diff --git a/test/user-turn-kind.mjs b/test/user-turn-kind.mjsindex 827011f53..7edcb922b 100644--- a/test/user-turn-kind.mjs+++ b/test/user-turn-kind.mjs@@ -85,7 +85,6 @@ for (const build of ['chrome', 'firefox']) {assert.equal((await f.mark('Please continue from where you left off.', 'act', { trustedContinuation: true })).webbrainTurnKind, 'continue');assert.equal((await f.mark('devam et', 'ask')).webbrainTurnKind, 'request', 'Ask turns are requests');assert.equal((await f.mark('devam et', 'act', {}, { sourceBound: true })).webbrainTurnKind, 'request', 'Shortcuts are requests');- assert.equal((await f.mark('x'.repeat(6001))).webbrainTurnKind, 'request', 'Long turns are requests');assert.equal((await f.mark('[Scheduled resume] check the queue')).webbrainTurnKind, 'request', 'Scheduled resumes keep their own rule');f.agent.conversations.set(f.tab, [{ role: 'system', content: 'system' }]);assert.equal((await f.mark('devam et')).webbrainTurnKind, 'request', 'A first turn has nothing to continue');