Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Publication prefetch: decision-model reads hold their cost; stale caption answers are dropped
Barack Sokullue82c8a58248c
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex dd919e453..416716cbc 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -9067,6 +9067,9 @@ export class Agent extends LoopDetector {request: trusted, items: indexes.map(index => data[index]) })), 12000)) },questions: semanticSystemOneBatchQuestions(kind, indexes.length),}, context);+ // A background read whose run ended or moved on while the request+ // was in flight keeps nothing from it.+ if (background && stopped()) break;let sure = 0;indexes.forEach((index, position) => {const answer = verdict.answers?.[`item_${position}`];@@ -9096,6 +9099,7 @@ export class Agent extends LoopDetector {background ? background.costState : (this.currentCostState.get(tabId) || null),{ tabId, generationName: `semantic_${kind}`, ...(background ? { background: true } : {}) });}, runSignal, mainTimeoutMs);+ if (background && stopped()) break;const accepted = acceptSemanticBatchAnswers(kind,this._socialPublicationResponseObject(response, { maxTokens: 120 + 60 * indexes.length }), trusted,indexes.map(index => data[index]));@@ -9145,6 +9149,8 @@ export class Agent extends LoopDetector {const askMain = mainModel && !this._workflowReplayTabs?.has(tabId);const readings = await this._semanticDecisions(tabId, kind, { items: pending.map(text => ({ ...context, text })) },{ ...(mainTimeoutMs ? { mainTimeoutMs } : {}), mainModel: askMain, ...(background ? { background } : {}) });+ // A background read whose run ended or moved on meanwhile stores nothing.+ if (background && (background.signal.aborted || !background.current())) return;// A text counts as unanswered only once the active model had its turn// too: a read by the decision model alone, a stopped run or a provider// that cannot answer leaves it for the next read. So does a background@@ -9329,25 +9335,39 @@ export class Agent extends LoopDetector {async evaluateSystemOne(tabId, client, args, context = this.systemOneContext(tabId)) {const costState = context.costState || this.currentCostState.get(tabId) || this._newCostRunState();+ const costProvider = args.config || SYSTEM_ONE_COST_PROVIDER;const activeConfig = this._activeProvider(tabId)?.config;const compass = args.config?.provider === 'compass' && activeConfig?.providerName === 'webbrain-cloud' ? activeConfig : null;- return client.evaluate({ ...args,- ...(compass ? {- headers: { 'X-WebBrain-Device-Id': compass.deviceGuid || '', 'X-WebBrain-Client': 'extension',- 'X-WebBrain-Help-Improve': compass.helpImproveWebBrain === false ? '0' : '1', ...args.headers },- metadata: { session_id: this.conversationIds.get(tabId), ...args.metadata },- } : {}),- beforeRequest: async () => {- if (this.strictSecretMode || (!args.config?.local && globalThis.navigator?.onLine === false) || context.isCurrent?.() === false) throw new Error('Jev unavailable.');- const message = await this._checkCostAllowance(args.config || SYSTEM_ONE_COST_PROVIDER, costState);- if (message) throw this._costAllowanceError(message);- },- onUsage: async metadata => {- const stopped = await this._recordCostUsage(args.config || SYSTEM_ONE_COST_PROVIDER, metadata.usage, costState);- this.recordSystemOneVerdict(tabId, { decision: 'usage', ...metadata }, context);- if (stopped) throw this._costAllowanceError(stopped);- },- });+ // Held at its most until it settles, like any other metered request+ // (_checkCostAllowance): its state and questions as the prompt and a short+ // answer per question. A background check's request (`context.background`)+ // is held as a background one.+ let hold = null;+ try {+ return await client.evaluate({ ...args,+ ...(compass ? {+ headers: { 'X-WebBrain-Device-Id': compass.deviceGuid || '', 'X-WebBrain-Client': 'extension',+ 'X-WebBrain-Help-Improve': compass.helpImproveWebBrain === false ? '0' : '1', ...args.headers },+ metadata: { session_id: this.conversationIds.get(tabId), ...args.metadata },+ } : {}),+ beforeRequest: async () => {+ if (this.strictSecretMode || (!args.config?.local && globalThis.navigator?.onLine === false) || context.isCurrent?.() === false) throw new Error('Jev unavailable.');+ hold ??= this._holdCostAllowance(costProvider,+ [{ role: 'user', content: JSON.stringify({ state: args.state ?? null, questions: args.questions ?? null }) }],+ { maxTokens: 256 + 128 * Object.keys(args.questions || {}).length }, costState,+ { background: context.background === true });+ const message = await this._checkCostAllowance(costProvider, costState, { signal: args.signal || null, hold });+ if (message) throw this._costAllowanceError(message);+ },+ onUsage: async metadata => {+ const stopped = await this._recordCostUsage(costProvider, metadata.usage, costState);+ this.recordSystemOneVerdict(tabId, { decision: 'usage', ...metadata }, context);+ if (stopped) throw this._costAllowanceError(stopped);+ },+ });+ } finally {+ hold?.release();+ }}// Where a completion check spent its time, for traces: preparing the@@ -23880,9 +23900,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d_invalidateSocialPublicationContract(guard) {// A contract compiled in the background before a correction is never- // installed after it, so its request is cancelled too, and so is a- // background audit against the contract the correction replaces.+ // installed after it, so its request is cancelled too, and so are a+ // background audit against the contract the correction replaces and the+ // caption readings the same prefetch started.this._dropSocialPublicationContractPrefetch(guard);+ for (const abort of this._socialPublicationCaptionReads?.get(guard) || []) {+ if (!abort.controller.signal.aborted) abort.controller.abort();+ }const social = guard.socialPublication;this._dropSocialPublicationAuditPrefetch(social);if (!social) return;@@ -24941,6 +24965,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!effects.length && !controls.length) return Promise.resolve([]);const abort = this._socialPublicationBackgroundAbort(binding, tabId);const background = this._socialPublicationBackgroundCall(tabId, guard, binding, abort);+ // A correction or clarification cancels them with the contract and the+ // audit (_invalidateSocialPublicationContract).+ const captionReads = guard ? ((this._socialPublicationCaptionReads ||= new WeakMap()).get(guard)+ || this._socialPublicationCaptionReads.set(guard, new Set()).get(guard)) : null;+ captionReads?.add(abort);const read = (kind, texts, request) => {for (const text of texts) reading.add(kind + '\u0000' + text);return Promise.resolve().then(request)@@ -24952,7 +24981,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreads.push(read('message_control', controls, () => this._readPageTexts(tabId, 'message_control', controls, {},{ maxChars: 400, mainTimeoutMs: CONTROL_READING_TIMEOUT_MS, background })));}- return this._socialPublicationBackgroundSettle(abort, Promise.allSettled(reads), []);+ return this._socialPublicationBackgroundSettle(abort, Promise.allSettled(reads), [])+ .finally(() => captionReads?.delete(abort));}_socialPublishedContractMatches(binding, guard, pageState) {diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 87f67bc5a..c65fc6670 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -8780,6 +8780,9 @@ export class Agent extends LoopDetector {request: trusted, items: indexes.map(index => data[index]) })), 12000)) },questions: semanticSystemOneBatchQuestions(kind, indexes.length),}, context);+ // A background read whose run ended or moved on while the request+ // was in flight keeps nothing from it.+ if (background && stopped()) break;let sure = 0;indexes.forEach((index, position) => {const answer = verdict.answers?.[`item_${position}`];@@ -8809,6 +8812,7 @@ export class Agent extends LoopDetector {background ? background.costState : (this.currentCostState.get(tabId) || null),{ tabId, generationName: `semantic_${kind}`, ...(background ? { background: true } : {}) });}, runSignal, mainTimeoutMs);+ if (background && stopped()) break;const accepted = acceptSemanticBatchAnswers(kind,this._socialPublicationResponseObject(response, { maxTokens: 120 + 60 * indexes.length }), trusted,indexes.map(index => data[index]));@@ -8858,6 +8862,8 @@ export class Agent extends LoopDetector {const askMain = mainModel && !this._workflowReplayTabs?.has(tabId);const readings = await this._semanticDecisions(tabId, kind, { items: pending.map(text => ({ ...context, text })) },{ ...(mainTimeoutMs ? { mainTimeoutMs } : {}), mainModel: askMain, ...(background ? { background } : {}) });+ // A background read whose run ended or moved on meanwhile stores nothing.+ if (background && (background.signal.aborted || !background.current())) return;// A text counts as unanswered only once the active model had its turn// too: a read by the decision model alone, a stopped run or a provider// that cannot answer leaves it for the next read. So does a background@@ -9042,25 +9048,39 @@ export class Agent extends LoopDetector {async evaluateSystemOne(tabId, client, args, context = this.systemOneContext(tabId)) {const costState = context.costState || this.currentCostState.get(tabId) || this._newCostRunState();+ const costProvider = args.config || SYSTEM_ONE_COST_PROVIDER;const activeConfig = this._activeProvider(tabId)?.config;const compass = args.config?.provider === 'compass' && activeConfig?.providerName === 'webbrain-cloud' ? activeConfig : null;- return client.evaluate({ ...args,- ...(compass ? {- headers: { 'X-WebBrain-Device-Id': compass.deviceGuid || '', 'X-WebBrain-Client': 'extension',- 'X-WebBrain-Help-Improve': compass.helpImproveWebBrain === false ? '0' : '1', ...args.headers },- metadata: { session_id: this.conversationIds.get(tabId), ...args.metadata },- } : {}),- beforeRequest: async () => {- if (this.strictSecretMode || (!args.config?.local && globalThis.navigator?.onLine === false) || context.isCurrent?.() === false) throw new Error('Jev unavailable.');- const message = await this._checkCostAllowance(args.config || SYSTEM_ONE_COST_PROVIDER, costState);- if (message) throw this._costAllowanceError(message);- },- onUsage: async metadata => {- const stopped = await this._recordCostUsage(args.config || SYSTEM_ONE_COST_PROVIDER, metadata.usage, costState);- this.recordSystemOneVerdict(tabId, { decision: 'usage', ...metadata }, context);- if (stopped) throw this._costAllowanceError(stopped);- },- });+ // Held at its most until it settles, like any other metered request+ // (_checkCostAllowance): its state and questions as the prompt and a short+ // answer per question. A background check's request (`context.background`)+ // is held as a background one.+ let hold = null;+ try {+ return await client.evaluate({ ...args,+ ...(compass ? {+ headers: { 'X-WebBrain-Device-Id': compass.deviceGuid || '', 'X-WebBrain-Client': 'extension',+ 'X-WebBrain-Help-Improve': compass.helpImproveWebBrain === false ? '0' : '1', ...args.headers },+ metadata: { session_id: this.conversationIds.get(tabId), ...args.metadata },+ } : {}),+ beforeRequest: async () => {+ if (this.strictSecretMode || (!args.config?.local && globalThis.navigator?.onLine === false) || context.isCurrent?.() === false) throw new Error('Jev unavailable.');+ hold ??= this._holdCostAllowance(costProvider,+ [{ role: 'user', content: JSON.stringify({ state: args.state ?? null, questions: args.questions ?? null }) }],+ { maxTokens: 256 + 128 * Object.keys(args.questions || {}).length }, costState,+ { background: context.background === true });+ const message = await this._checkCostAllowance(costProvider, costState, { signal: args.signal || null, hold });+ if (message) throw this._costAllowanceError(message);+ },+ onUsage: async metadata => {+ const stopped = await this._recordCostUsage(costProvider, metadata.usage, costState);+ this.recordSystemOneVerdict(tabId, { decision: 'usage', ...metadata }, context);+ if (stopped) throw this._costAllowanceError(stopped);+ },+ });+ } finally {+ hold?.release();+ }}// Where a completion check spent its time, for traces: preparing the@@ -21063,9 +21083,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d_invalidateSocialPublicationContract(guard) {// A contract compiled in the background before a correction is never- // installed after it, so its request is cancelled too, and so is a- // background audit against the contract the correction replaces.+ // installed after it, so its request is cancelled too, and so are a+ // background audit against the contract the correction replaces and the+ // caption readings the same prefetch started.this._dropSocialPublicationContractPrefetch(guard);+ for (const abort of this._socialPublicationCaptionReads?.get(guard) || []) {+ if (!abort.controller.signal.aborted) abort.controller.abort();+ }const social = guard.socialPublication;this._dropSocialPublicationAuditPrefetch(social);if (!social) return;@@ -22124,6 +22148,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!effects.length && !controls.length) return Promise.resolve([]);const abort = this._socialPublicationBackgroundAbort(binding, tabId);const background = this._socialPublicationBackgroundCall(tabId, guard, binding, abort);+ // A correction or clarification cancels them with the contract and the+ // audit (_invalidateSocialPublicationContract).+ const captionReads = guard ? ((this._socialPublicationCaptionReads ||= new WeakMap()).get(guard)+ || this._socialPublicationCaptionReads.set(guard, new Set()).get(guard)) : null;+ captionReads?.add(abort);const read = (kind, texts, request) => {for (const text of texts) reading.add(kind + '\u0000' + text);return Promise.resolve().then(request)@@ -22135,7 +22164,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreads.push(read('message_control', controls, () => this._readPageTexts(tabId, 'message_control', controls, {},{ maxChars: 400, mainTimeoutMs: CONTROL_READING_TIMEOUT_MS, background })));}- return this._socialPublicationBackgroundSettle(abort, Promise.allSettled(reads), []);+ return this._socialPublicationBackgroundSettle(abort, Promise.allSettled(reads), [])+ .finally(() => captionReads?.delete(abort));}_socialPublishedContractMatches(binding, guard, pageState) {diff --git a/test/publication-prefetch.mjs b/test/publication-prefetch.mjsindex 5e17283e0..50eb290a9 100644--- a/test/publication-prefetch.mjs+++ b/test/publication-prefetch.mjs@@ -990,6 +990,79 @@ for (const browser of ['chrome', 'firefox']) {assert.equal(traced.every(context => context?.runId === 'run-A'), true, 'traced only to the run that started it');});+ test(`${browser}: a decision-model request holds its cost until it settles, as a background one for a background read`, async () => {+ const agent = new Agent({});+ const tabId = 982;+ agent._getCostAllowanceState = async () => ({ sessionLimitUsd: 1, totalLimitUsd: 100, totalSpentUsd: 0 });+ agent._recordCostUsage = async () => null;+ const config = { provider: 'typesafe', config: { category: 'cloud', inputCostPerMillionUsd: 1, outputCostPerMillionUsd: 1 } };+ const runCost = { spentUsd: 0 };+ const held = [];+ const client = { evaluate: async ({ beforeRequest, onUsage }) => {+ await beforeRequest();+ held.push([...(agent._costHolds || [])].map(hold => ({ background: hold.background, costState: hold.costState, usd: hold.usd })));+ await onUsage({ usage: { input_tokens: 10, output_tokens: 1 } });+ return { answers: {} };+ } };+ const args = { config, state: { context: 'x'.repeat(4000) }, questions: { item_0: {} } };+ await agent.evaluateSystemOne(tabId, client, args, { costState: runCost, runId: 'run-A', isCurrent: () => true, background: true });+ await agent.evaluateSystemOne(tabId, client, args, { costState: runCost, runId: 'run-A', isCurrent: () => true });+ assert.equal(held.length, 2);+ assert.deepEqual(held.map(list => list.length), [1, 1], 'one hold per request in flight');+ assert.equal(held[0][0].background, true, 'a background read is held as a background request');+ assert.equal(held[0][0].costState, runCost, 'against the allowance of its run');+ assert.ok(held[0][0].usd > 0, 'at its most');+ assert.equal(held[1][0].background, false);+ assert.equal(agent._costHolds?.size || 0, 0, 'released once the request settled');+ });++ test(`${browser}: a correction cancels the prefetch's caption reads, and an answer that outlived its run is kept nowhere`, async () => {+ // A correction cancels the reads the prefetch started, with its contract and audit.+ {+ const f = setup();+ const { agent, tabId } = f;+ agent._runAbortStates.set(tabId, { controller: new AbortController(), dispose() {} });+ f.detected.publicationPrefetchCaptions = ['Post', 'Close'];+ const reads = [];+ agent._readPageTexts = (_tabId, kind, texts, _context, options = {}) => new Promise(resolve => {+ reads.push({ kind, background: options.background, resolve });+ });+ const prefetch = f.write();+ await until(() => reads.length === 2, 'both caption reads started');+ f.applySteering('Make it shorter.');+ for (const { kind, background } of reads) assert.equal(background.signal.aborted, true, `${kind}: cancelled by the correction`);+ await within(prefetch, 'the prefetch settles');+ for (const read of reads) read.resolve();+ }+ // An answer that arrives after its run moved on is neither returned,+ // cached nor stored as a reading: the next read asks again.+ {+ const provider = { name: 'checker', model: 'checker', chat: async () => ({ content: '{}' }) };+ const agent = new Agent({ getActive: () => provider });+ const tabId = 983;+ agent._jevSettings = async () => null;+ let live = true;+ let sent = 0;+ agent._chatWithCostAllowance = async () => {+ sent += 1;+ live = false; // the run moves on while the request is in flight+ return { content: JSON.stringify({ answers: [{ item: 0, choice: 'publish' }] }) };+ };+ const background = { signal: new AbortController().signal, costState: { spentUsd: 0 }, runId: 'run-A', current: () => live };+ const [late] = await agent._semanticDecisions(tabId, 'control_effect', { items: [{ text: 'Post' }] }, { provider, background });+ assert.equal(late.choice, null, 'the late answer is dropped');+ live = true;+ await agent._readPageTexts(tabId, 'control_effect', ['Post'], {}, { background });+ assert.equal(sent, 2, 'nothing was cached, so the next read asks again');+ assert.equal(agent._pageTextReading('control_effect', 'Post'), null, 'nor is the late answer stored as a reading');+ // The same answer within a live run is kept as before.+ live = true;+ agent._chatWithCostAllowance = async () => { sent += 1; return { content: JSON.stringify({ answers: [{ item: 0, choice: 'publish' }] }) }; };+ await agent._readPageTexts(tabId, 'control_effect', ['Post'], {}, { background });+ assert.equal(agent._pageTextReading('control_effect', 'Post'), 'publish');+ }+ });+test(`${browser}: an upload into the composer last written audits the draft again`, async () => {const f = setup({ request: 'Tweet about my blog post',contract: rawContract([rawAction('p1', 'twitter', 'about my blog post', 'compose')]) });