Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

23.0 KB
Raw Blame
import { strict as assert } from 'node:assert';
import { test } from 'node:test';
import { readFileSync } from 'node:fs';
import { chromium, firefox } from 'playwright';
import { createCompletionInvariantState } from '../src/chrome/src/agent/completion-invariant.js';
import { defaultDecide, semanticQuestionOf } from './semantic-decision-stub.mjs';

// The page, native AX refs, submit probe, normalized intent contract, and
// content dispatch are real. Every browser request is fulfilled locally.
const noneContract = {version:1,status:'none',actions:[],requirements:null,prohibited:[],reason:'No publication requested.'};

for (const build of ['chrome','firefox']) {
  test(`${build}: ordinary UI activation respects the current publication obligation`, async t => {
    const {Agent} = await import(`../src/${build}/src/agent/agent.js`);
    const browser = await (build==='chrome'?chromium:firefox).launch({headless:true});
    const context = await browser.newContext();
    let requests = 0;
    await context.route('**/*', route => {requests++; return route.fulfill({contentType:'text/html',body:'<!doctype html>'});});
    const page = await context.newPage();
    await page.goto('https://bsky.app/');
    await page.evaluate(() => {
      window.contentListeners=[];
      window.chrome={runtime:{sendMessage:()=>{},onMessage:{addListener:fn=>window.contentListeners.push(fn)}}};
      window.browser=window.chrome;
    });
    for(const name of ['accessibility-tree.js','content.js'])await page.addScriptTag({content:readFileSync(`src/${build}/src/content/${name}`,'utf8')});
    const call = (action, params) => page.evaluate(({action,params}) => new Promise(resolve => {
      for(const listener of window.contentListeners) listener({target:'content',action,params},{},resolve);
    }),{action,params});
    const originalChrome=globalThis.chrome;
    const originalBrowser=globalThis.browser;
    if(build==='chrome')globalThis.chrome={scripting:{executeScript:async({func,args})=>[{frameId:0,result:await page.evaluate(({source,args})=>Function('return ('+source+')')()(...args),{source:func.toString(),args})}]}};
    else{
      globalThis.chrome=undefined;
      globalThis.browser={tabs:{executeScript:async(_tab,{code})=>[await page.evaluate(code=>Function('return ('+code+')')(),code)]}};
    }
    try {
      const setup = async (html, options={}) => {
        await page.setContent('<style>button,[role=button],a{display:inline-block;min-width:70px;min-height:30px} [hidden]{display:none}</style>'+html);
        await page.evaluate(() => {
          window.uiClicks=[];
          document.addEventListener('click',event=>{
            const control=event.target.closest('button,[role=button],a');
            if(control){event.preventDefault();window.uiClicks.push(control.id);if(control.id==='dismiss')control.closest('[role=dialog]')?.remove();}
          });
        });
        const provider={model:'owned-fixture-provider',chat:async()=>({content:'{}'})};
        const agent=new Agent({getActive:()=>provider});
        agent._currentUrl=async()=>page.url();
        agent._persist=()=>{};
        agent.conversations.set(1,[{role:'system',content:'system'},{role:'user',content:options.request||'Inspect the current UI and dismiss its informational dialog. Send and publish nothing.'}]);
        const guard=agent._startPlanExecutionGuard(1,'act',{requestKind:'execute',requiresStateChange:false,requiresSubmission:false,...options.gate});
        // Fixed-choice decisions about the request get a model's reading; the
        // publication contract comes from the fixture.
        agent._chatWithCostAllowance=async(_p,messages,_o,_c,meta)=>{
          const kind=String(meta?.generationName||'').match(/^semantic_(.+)$/)?.[1];
          return {content:JSON.stringify(kind?defaultDecide(kind,semanticQuestionOf(agent,messages)):(options.contract||noneContract))};
        };
        if(options.compile!==false)await agent._ensureSocialPublicationContract(1,provider);
        return {agent,guard,provider};
      };
      const attempt = async (f,args) => {
        const detected=await f.agent._detectLikelySubmitAction(1,'click_ax',args);
        const blocked=await f.agent._socialPublicationPreSubmitBlock(1,'click_ax',args,detected,f.provider);
        if(blocked)return {detected,result:blocked};
        return {detected,result:await call('click_ax',args)};
      };
      const targetRef = async id => {
        const tree=await call('get_accessibility_tree',{filter:'all',maxChars:16000});
        assert.equal(typeof tree.pageContent,'string',JSON.stringify(tree));
        return page.evaluate(id=>window.__wb_ax_ref(document.getElementById(id)),id);
      };

      await t.test('observed native verification modal Close is dispatched and the dialog actually disappears',async()=>{
        const f=await setup('<div role="dialog" aria-label="Make adjustments to email settings for your account"><button id="verify" type="button">Send verification email</button><button id="dismiss" type="button" aria-label="Close active dialog"></button></div>');
        const ref=await targetRef('dismiss');
        const {detected,result}=await attempt(f,{ref_id:ref});
        assert.equal(detected?.resolvedOrdinaryActivationTarget,true);
        assert.equal(result.success,true);
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),['dismiss']);
        assert.equal(await page.locator('[role=dialog]').count(),0);
      });

      for(const [label,tag]of [['Show details','button'],['Expand filters','div'],['Dismiss notice','button']]){
        await t.test(`ordinary ${label} is not forced into a publication contract`,async()=>{
          const f=await setup(`<${tag} id="ordinary" ${tag==='button'?'type="button"':'role="button" tabindex="0"'}>${label}</${tag}>`);
          const {detected,result}=await attempt(f,{ref_id:await targetRef('ordinary')});
          assert.equal(detected?.resolvedOrdinaryActivationTarget,true);
          assert.equal(result.success,true);
          assert.deepEqual(await page.evaluate(()=>window.uiClicks),['ordinary']);
        });
      }

      const unsafeControls=[
        ['native publish','<div><textarea>Draft</textarea><button id="unsafe" type="button" data-testid="composerPublishBtn">Post</button></div>'],
        ['publish without an editor','<button id="unsafe" type="button" data-testid="composerPublishBtn">Post</button>'],
        ['native send','<button id="unsafe" type="button" data-testid="sendMessageButton">Send message</button>'],
        ...['send','sendMessageButton','dmComposerSendButton','send-message-button'].map(marker=>[`unnamed native send ${marker}`,`<button id="unsafe" type="button" data-testid="${marker}"></button>`]),
        ['unnamed unresolved purpose','<button id="unsafe" type="button"></button>'],
        ['send verification email','<div role="dialog"><button id="unsafe" type="button">Send verification email</button></div>'],
        ['custom submit attribute','<button id="unsafe" type="button" onclick="document.querySelector(\'form\').requestSubmit()">Finish</button><form></form>'],
        ['custom action','<button id="unsafe" type="button" data-action="submit">Finish</button>'],
        ['conflicting accessible commit name','<button id="unsafe" type="button" aria-label="Send">Show details</button>'],
        ['explicit submit type','<button id="unsafe" type="submit">Finish</button>'],
        ['hidden target','<button id="unsafe" type="button" hidden>Show details</button>'],
        ['unbound saved resource deletion','<button id="unsafe" type="button">Delete resource</button>'],
      ];
      for(const[name,html]of unsafeControls)await t.test(`${name} retains no-dispatch protection`,async()=>{
        const f=await setup(html);
        const ref=await targetRef('unsafe');
        const {detected,result}=await attempt(f,{ref_id:ref});
        assert.notEqual(detected?.resolvedOrdinaryActivationTarget,true);
        assert.equal(result.noDispatch,true);
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),[]);
      });

      await t.test('associated forms retain their existing generic submission classification',async()=>{
        const f=await setup('<form id="f"><input name="value"></form><button id="unsafe" type="button" form="f">Finish</button>');
        const detected=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('unsafe')});
        assert.equal(detected?.isSubmit,true);
        assert.equal(detected?.publicationControl,false);
        assert.notEqual(detected?.resolvedOrdinaryActivationTarget,true);
        // The social guard delegates this already classified ordinary form to
        // the generic form/confirmation gates; it is not the new UI exemption.
        assert.equal(await f.agent._socialPublicationPreSubmitBlock(1,'click_ax',{},detected,f.provider),null);
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),[]);
      });

      for(const[condition,options,change]of[
        ['missing contract',{compile:false}],
        ['ambiguous contract',{contract:{...noneContract,status:'clarify',reason:'Intent ambiguous.'}}],
        ['submission obligation',{gate:{requiresSubmission:true}}],
        ['unknown submission obligation',{gate:{requiresSubmission:null}}],
        ['ready publication contract',{request:'Post Hello on Bluesky.',contract:{version:1,status:'ready',actions:[{id:'p1',platform:'bluesky',account:null,posts:[{body:{kind:'exact',source:{source:'request',start:'Hello',end:'Hello'}},media:{kind:'count',type:'any',format:null,min:0,max:0},context:{kind:'post',target:null}}]}],requirements:'p1',prohibited:[],reason:'Publication requested.'}}],
        ['contract requires refresh',{},f=>{f.guard.socialPublication.needsRecompile=true;}],
      ])await t.test(`${condition} does not turn a resolved preparatory activation into publication`,async()=>{
        const f=await setup('<button id="ordinary" type="button">Show details</button>',options);
        change?.(f);
        const {result}=await attempt(f,{ref_id:await targetRef('ordinary')});
        assert.equal(result.success,true);
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),['ordinary']);
      });
      for(const[condition,options,change]of[
        ['different task identity',{},f=>{f.guard.taskKey='replacement-task';}],
        ['Stop',{},f=>{f.agent._checkAbort=()=>true;}],
        ['pending accepted steering',{},f=>{f.agent._hasPendingSteering=()=>true;}],
        ['task replacement during URL read',{},f=>{f.agent._currentUrl=async()=>{f.agent._planExecutionGuards.set(1,{...f.guard});return page.url();};}],
      ])await t.test(`${condition} cannot exempt an ordinary activation`,async()=>{
        const f=await setup('<button id="ordinary" type="button">Show details</button>',options);
        change?.(f);
        const {result}=await attempt(f,{ref_id:await targetRef('ordinary')});
        assert.equal(result.noDispatch,true);
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),[]);
      });

      await t.test('missing refs and ambiguous text never manufacture a resolved target',async()=>{
        const f=await setup('<button type="button">Show details</button><button type="button">Show details</button>');
        for(const[tool,args]of [['click_ax',{ref_id:'ref_999999999999'}],['click',{text:'Show details',textMatch:'exact'}]]){
          const detected=await f.agent._detectLikelySubmitAction(1,tool,args);
          assert.notEqual(detected?.resolvedOrdinaryActivationTarget,true);
          assert.equal((await f.agent._socialPublicationPreSubmitBlock(1,tool,args,detected,f.provider)).noDispatch,true);
        }
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),[]);
      });
      await t.test('structural resource popup labels pass with no publication contract and after steering',async()=>{
        for(const host of ['https://example.test/','https://x.com/','https://bsky.app/']) {
          await page.goto(host);
          await page.evaluate(() => { window.contentListeners=[]; window.chrome={runtime:{sendMessage:()=>{},onMessage:{addListener:fn=>window.contentListeners.push(fn)}}};window.browser=window.chrome; });
          for(const name of ['accessibility-tree.js','content.js'])await page.addScriptTag({content:readFileSync(`src/${build}/src/content/${name}`,'utf8')});
          const f=await setup('<button id="ordinary" type="button" aria-haspopup="menu" aria-controls="choices">Thread post options</button><div id="choices" role="menu"></div>',{compile:false});
          f.guard.steeringPriorSubmission={dispatched:true,observedAfterSubmit:false};
          for(const [tool,args]of [['click_ax',{ref_id:await targetRef('ordinary')}],['click',{selector:'#ordinary'}]]) {
            const detected=await f.agent._detectLikelySubmitAction(1,tool,args);
            assert.equal(detected.resolvedPreparatoryTarget,true);
            assert.equal(f.agent._steeringPriorSubmissionBlock(1,tool,args,detected),null);
            assert.equal(await f.agent._socialPublicationPreSubmitBlock(1,tool,args,detected,f.provider),null);
          }
        }
      });
      await t.test('compound and localized commit captions do not gain ordinary activation status',async()=>{
        for(const caption of ['Click here to publish','Schedule and publish','Review then delete','Click to confirm','Jetzt senden','Şimdi gönder']) {
          const f=await setup(`<button id="unsafe" type="button">${caption}</button>`,{compile:false});
          const detected=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('unsafe')});
          assert.notEqual(detected?.resolvedOrdinaryActivationTarget,true,caption);
        }
      });
      await t.test('native resource scope ignores unrelated height but rejects virtualization, hiding and replaced owners',async()=>{
        const url='https://bsky.app/profile/alice/post/900';
        for(const failure of [null,'virtualized','hidden','replaced-owner','expired']) {
          const f=await setup(`<section id="collection"><article id="resource" ${failure==='virtualized'?'aria-setsize="200" aria-posinset="1"':''}><a rel="bookmark" href="${url}">Original</a><p>Saved exact body</p><button id="ordinary" type="button" aria-haspopup="menu">Resource controls</button></article></section><aside style="height:3000px">Unrelated sidebar</aside><form><button id="finish" type="submit">Save</button></form>`,{compile:false});
          const observed=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('ordinary')});
          assert.equal(observed.actionResource?.complete,true);
          f.guard.steeringPriorSubmissionReconciliation={resourceScopeToken:observed.actionResource.scopeToken,resourceUrl:url};
          await page.evaluate(({failure,token})=>{const article=document.getElementById('resource');
            if(failure==='hidden')article.hidden=true;
            else if(failure==='replaced-owner')article.parentElement.outerHTML='<section id="collection"></section>';
            else article.remove();
            if(failure==='expired')globalThis.__wbResourceMutationScopes.scopes.get(token).until=0;
          },{failure,token:observed.actionResource.scopeToken});
          const fresh=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('finish')},{mutationReconciliation:true});
          assert.equal(fresh.reconciliationResourceScanComplete,false,'Whole-page height cannot prove absence');
          assert.equal(fresh.reconciliationScope?.removed===true,failure===null,String(failure));
        }
      });
      await t.test('native draft toolbar and selected local attachment controls remain preparation',async()=>{
        const f=await setup('<section><div role="toolbar" aria-controls="draft"><button id="ordinary" type="button">Remove formatting</button></div><div id="draft" contenteditable="true">Local draft</div></section>',{compile:false});
        let detected=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('ordinary')});
        assert.equal(detected.resolvedNonSubmitTarget,true);
        assert.equal(detected.mutationKind,undefined);
        assert.equal(await f.agent._socialPublicationPreSubmitBlock(1,'click_ax',{},detected,f.provider),null);
        const attached=await setup('<div role="dialog"><div contenteditable="true">Local draft</div><input id="file" type="file"><figure><img id="preview" style="width:30px;height:30px"><button id="ordinary" type="button">Remove attachment</button></figure></div>',{compile:false});
        await page.locator('#file').setInputFiles({name:'local.txt',mimeType:'text/plain',buffer:Buffer.from('local fixture')});
        await page.evaluate(()=>{document.getElementById('preview').src=URL.createObjectURL(document.getElementById('file').files[0]);});
        detected=await attached.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('ordinary')});
        assert.equal(detected.resolvedNonSubmitTarget,true);
        assert.equal(detected.mutationKind,undefined);
        assert.equal(await attached.agent._socialPublicationPreSubmitBlock(1,'click_ax',{},detected,attached.provider),null);
      });
      await t.test('form editor ownership cannot turn Save or resource deletion into local formatting',async()=>{
        const f=await setup('<form><div role="toolbar" aria-controls="draft"><button id="bold" type="button">Bold</button><button id="format" type="button">Remove formatting</button><button id="save" type="button">Save</button><button id="delete" type="button">Delete record</button><button id="remove" type="button">Remove record</button><button id="title-save" type="button" title="Save record">Bold</button><button id="title-delete" type="button" title="Delete record">Bold</button></div><div id="draft" contenteditable="true">Local draft</div></form>',{compile:false});
        f.guard.steeringPriorSubmission={dispatched:true,observedAfterSubmit:false};
        for(const id of ['bold','format']) {
          const detected=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef(id)});
          assert.equal(detected.isSubmit,false,id);assert.equal(detected.resolvedNonSubmitTarget,true,id);
          assert.equal(f.agent._steeringPriorSubmissionBlock(1,'click_ax',{},detected),null,id);
        }
        for(const id of ['save','delete','remove','title-save','title-delete']) {
          const detected=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef(id)});
          assert.equal(detected.isSubmit,true,id);assert.notEqual(detected.resolvedDraftControl,true,id);
          assert.equal(f.agent._steeringPriorSubmissionBlock(1,'click_ax',{},detected).noDispatch,true,id);
        }
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),[]);
      });
      await t.test('resource B never borrows A and portal ownership requires its actual clicked native launcher',async()=>{
        const url='https://bsky.app/profile/alice/post/900';
        const f=await setup(`<section><article id="owned"><a rel="bookmark" href="${url}">Original</a><p>Saved body</p><button id="launcher" type="button" aria-haspopup="menu" aria-controls="choices">Resource controls</button></article><article id="other"><p>Unbound resource B</p><button id="wrong" type="button">Delete record</button></article></section><div id="choices" role="menu" aria-labelledby="launcher"><button id="portal" type="button">Delete record</button></div><div id="unrelated" role="menu"><button id="foreign" type="button">Delete record</button></div>`,{compile:false});
        f.agent.completionInvariants.set(1,createCompletionInvariantState('owned-resource-run'));
        let portal=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('portal')});
        assert.equal(portal.actionResource,undefined,'Read-only preparation does not establish portal ownership');
        await page.evaluate(()=>{document.getElementById('choices').hidden=true;document.getElementById('launcher').addEventListener('click',()=>{document.getElementById('choices').hidden=false;});});
        const args={ref_id:await targetRef('launcher')}, opened=await attempt(f,args);
        assert.equal(opened.result.success,true);
        f.agent._recordCompletionToolResult(1,'click_ax',args,opened.result,{detectedSubmit:opened.detected});
        assert.equal(f.guard.resolvedResourceScope.launcherDispatched,true);
        const wrongArgs={ref_id:await targetRef('wrong')}, wrong=await f.agent._detectLikelySubmitAction(1,'click_ax',wrongArgs);
        assert.equal(wrong.actionResource,undefined,'An existing different owner vetoes fallback, even if it lacks a permalink');
        // B is bound only to its own visible record, which this request never asks to delete.
        assert.equal(wrong.actionRecord?.text,'Unbound resource B');
        assert.equal((await f.agent._resourceMutationPreSubmitBlock(1,'click_ax',wrongArgs,wrong,f.provider)).actionGuard.reason,'deletion_not_requested');
        portal=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('portal')});
        assert.equal(portal.actionResource.url,url,'A native aria-controls/labelledby relation resolves the actual opened A menu');
        const foreign=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('foreign')});
        assert.equal(foreign.actionResource,undefined,'A different portal cannot borrow the latest URL');
        f.guard.resolvedResourceScope.launcherDispatched=false;
        portal=await f.agent._detectLikelySubmitAction(1,'click_ax',{ref_id:await targetRef('portal')});
        assert.equal(portal.actionResource,undefined,'Uncertain or undispatched launchers confer no ownership');
      });
      await t.test('unique text and selector resolve the same ordinary native activation',async()=>{
        const f=await setup('<button id="ordinary" type="button">Show details</button>');
        for(const args of [{text:'Show details',textMatch:'exact'},{selector:'#ordinary'}]){
          const detected=await f.agent._detectLikelySubmitAction(1,'click',args);
          assert.equal(detected?.resolvedOrdinaryActivationTarget,true);
          assert.equal(await f.agent._socialPublicationPreSubmitBlock(1,'click',args,detected,f.provider),null);
          assert.equal((await call('click',args)).success,true);
        }
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),['ordinary','ordinary']);
      });
      await t.test('no publication task still cannot bypass activation keys, callback tools, or social API writes',async()=>{
        const f=await setup('<button id="ordinary" type="button">Show details</button>');
        for(const[tool,args]of [['press_keys',{key:'Enter'}],['press_keys',{key:'Space'}],['execute_js',{code:'document.querySelector("button").click()'}],['execute_webmcp_tool',{name:'publish'}],['fetch_url',{url:'https://bsky.app/api/post',method:'POST',body:'{}'}]]){
          const detected=await f.agent._detectLikelySubmitAction(1,tool,args);
          assert.equal((await f.agent._socialPublicationPreSubmitBlock(1,tool,args,detected,f.provider)).noDispatch,true,tool);
        }
        assert.deepEqual(await page.evaluate(()=>window.uiClicks),[]);
      });
      assert.equal(requests,4,'only the intercepted fixture navigations occurred');
    }finally{globalThis.chrome=originalChrome;globalThis.browser=originalBrowser;await browser.close();}
  });
}