Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

22.6 KB
Raw Blame
import assert from 'node:assert/strict';
import { test } from 'node:test';
import { chromium, firefox } from 'playwright';
import { fixtureControlEffectReading, installModelStub, probeWithReadings } from './semantic-decision-stub.mjs';

// Mastodon 4.4 to 4.7 as a trace showed it, in a 1126 px wide tab beside
// WebBrain's panel. Mastodon renders its composer beside the timeline only from
// 1175 px, so there is none here; the signed-in account is in the navigation
// panel (since 4.4); a post's menu mounts at the end of the body, and its
// dangerous entries are Delete and Delete & re-draft; the delete confirmation
// opens a moment after the click, and while it is open ModalRoot marks every
// sibling of .modal-root inert. In the trace WebBrain opened the post's menu,
// chose Sil, and the confirmation that click opened was refused as "not inside
// one identifiable record".
const origin = 'https://mastoturk.org';
const own = id => `${origin}/@ownedqa/${id}`;
const status = (id, text, account = 'ownedqa') => `<article data-id="${id}"><div class="status__wrapper status__wrapper-public focusable" tabindex="0" aria-label="QA ${account}, ${text}, 15 Eki 21:01, ${account}">
<div class="status status-public" id="record-${id}"><div class="status__info"><a href="/@${account}/${id}" class="status__relative-time"><time>9m</time></a>
<a href="/@${account}" title="${account}" class="status__display-name"><span class="display-name"><bdi><strong class="display-name__html">QA ${account}</strong></bdi> <span class="display-name__account">@${account}</span></span></a></div>
<div class="status__content"><div class="status__content__text status__content__text--visible translate" lang="tr"><p>${text}</p></div></div>
<div class="status__action-bar"><div class="status__action-bar__button-wrapper"><button type="button" aria-label="Yanıtla" title="Yanıtla" class="status__action-bar__button icon-button"><svg class="icon icon-reply"></svg></button></div>
<div class="status__action-bar__button-wrapper"><button type="button" aria-label="Daha fazla" aria-expanded="false" title="Daha fazla" id="more-${id}" class="status__action-bar__button icon-button"><svg class="icon icon-ellipsis-h"></svg></button></div></div>
</div></div></article>`;
const accountBar = '<div class="navigation-bar"><div class="account account--minimal"><div class="account__wrapper"><div class="account__info-wrapper"><a class="account__display-name" title="ownedqa" href="/@ownedqa"><span class="display-name"><bdi><strong class="display-name__html">QA</strong></bdi> <span class="display-name__account">@ownedqa</span></span></a></div></div></div></div>';
const navigation = withAccount => `<div class="columns-area__panels__pane columns-area__panels__pane--start columns-area__panels__pane--navigational"><div class="columns-area__panels__pane__inner"><div class="navigation-panel">
<div class="navigation-panel__logo"><a href="/" class="column-link column-link--logo">Mastodon</a></div><form class="search"><input type="text" class="search__input" placeholder="Ara veya bağlantıyı yapıştır"></form>
${withAccount ? accountBar : ''}<div class="navigation-panel__menu"><a class="button navigation-panel__compose-button column-link" href="/publish">Yeni Gönderi</a><a class="column-link" href="/home">Anasayfa</a></div></div></div></div>`;
const composer = '<div class="columns-area__panels__pane columns-area__panels__pane--compositional"><div class="compose-panel"><form class="compose-form"><textarea class="autosuggest-textarea__textarea" placeholder="Aklında ne var?"></textarea></form></div></div>';
const menu = `<div id="menu-X" style="position:absolute;left:600px;top:120px;z-index:5;background:#fff"><div class="dropdown-animation dropdown-menu bottom"><div class="dropdown-menu__arrow bottom"></div><div class="dropdown-menu__container"><ul class="dropdown-menu__container__list">
<li class="dropdown-menu__item"><button data-index="0">Bu gönderiyi genişlet</button></li><li class="dropdown-menu__item"><button data-index="1">Gönderi bağlantısını kopyala</button></li>
<li class="dropdown-menu__separator"></li><li class="dropdown-menu__item"><button data-index="3" id="menu-edit">Düzenle</button></li>
<li class="dropdown-menu__item dropdown-menu__item--dangerous"><button data-index="4" id="menu-delete">Sil</button></li>
<li class="dropdown-menu__item dropdown-menu__item--dangerous"><button data-index="5" id="menu-redraft">Sil, Düzenle ve yeniden-paylaş</button></li></ul></div></div></div>`;
// Mastodon 4.4 and 4.5 render the confirmation as a div; 4.6 and later as a
// form whose confirm button submits it.
const confirmation = shell => {
  const body = '<div class="safety-action-modal__top"><div class="safety-action-modal__confirmation"><h1>Gönderiyi sil?</h1><p>Bu gönderiyi silmek istediğinden emin misin?</p></div></div>';
  return '<div style="pointer-events:auto"><div role="presentation" class="modal-root__overlay"></div><div role="dialog" class="modal-root__container">'
    + (shell === 'form'
      ? `<form class="modal-root__modal safety-action-modal">${body}<div class="safety-action-modal__bottom"><div class="safety-action-modal__actions"><button type="button" class="link-button" id="cancel">İptal</button><button type="submit" class="button" id="confirm">Sil</button></div></div></form>`
      : `<div class="modal-root__modal safety-action-modal">${body}<div class="safety-action-modal__bottom"><div class="safety-action-modal__actions"><button class="link-button" id="cancel">İptal</button><button type="button" class="button" id="confirm">Sil</button></div></div></div>`)
    + '</div></div>';
};
const page = ({ layout = 'narrow', shell = 'classic', deleteModal = true, signedIn = 'ownedqa', entries = 'buttons' } = {}) => `<!doctype html><meta charset="utf-8">
<style>body{margin:0;font:15px sans-serif}button{padding:8px}.columns-area__panels{display:flex}.columns-area__panels__main{width:600px}.navigation-panel{width:300px}
.modal-root__overlay{position:fixed;inset:0;background:rgba(0,0,0,.4)}.modal-root__container{position:fixed;inset:0;display:flex;align-items:center;justify-content:center}
.safety-action-modal{position:relative;background:#fff;padding:20px}svg{width:12px;height:12px;display:inline-block}</style>
<script id="initial-state" type="application/json">${JSON.stringify({ meta: { me: '1', delete_modal: deleteModal },
  accounts: { 1: { id: '1', username: signedIn, acct: signedIn, url: `${origin}/@${signedIn}` } } })}</script>
<div class="app-holder" id="mastodon"><div class="ui"><div class="columns-area__panels">${layout === 'wide' ? composer : ''}
<div class="columns-area__panels__main"><div class="column" role="region" aria-label="Anasayfa"><div role="feed" class="item-list">
${status('200', 'Test gönderisi / Test post')}${status('201', 'WebBrain test gönderisi')}${status('300', 'Başka birinin gönderisi', 'otherqa')}
</div></div></div>${navigation(layout !== 'hidden-account')}</div><div class="modal-root"></div></div></div>
<script>
window.qaDeletes = []; window.qaModal = ${JSON.stringify(confirmation(shell))};
window.qaMenu = ${JSON.stringify(entries === 'links' ? menu.replace(/<button data-index="(\d+)"( id="[^"]+")?>([^<]*)<\/button>/g, '<a href="#" role="button" tabindex="0" data-index="$1"$2>$3</a>') : menu)};
const ui = document.querySelector('.ui'), modalRoot = document.querySelector('.modal-root');
// ModalRoot: every sibling of .modal-root is inert while a modal is open.
const background = inert => { for (const sibling of ui.children) if (sibling !== modalRoot) inert ? sibling.setAttribute('inert', '') : sibling.removeAttribute('inert'); };
window.qaCloseModal = () => { modalRoot.innerHTML = ''; background(false); };
window.qaOpenModal = onConfirm => {
  modalRoot.innerHTML = qaModal; background(true);
  document.querySelector('#confirm').onclick = event => { event.preventDefault(); onConfirm(); qaCloseModal(); };
  document.querySelector('#cancel').onclick = event => { event.preventDefault(); qaCloseModal(); };
};
for (const id of ['200', '201', '300']) {
  const more = document.querySelector('#more-' + id);
  more.onclick = () => {
    if (document.querySelector('[id^="menu-2"],[id^="menu-3"]')) return;
    more.setAttribute('aria-expanded', 'true'); more.setAttribute('aria-controls', 'menu-' + id);
    more.querySelector('svg').setAttribute('class', 'icon icon-close');
    document.body.insertAdjacentHTML('beforeend', qaMenu.replace('menu-X', 'menu-' + id));
    const close = () => { document.querySelector('#menu-' + id)?.remove(); more.setAttribute('aria-expanded', 'false');
      more.querySelector('svg').setAttribute('class', 'icon icon-ellipsis-h'); };
    for (const item of document.querySelectorAll('#menu-' + id + ' [data-index]')) item.onclick = event => {
      event.preventDefault(); close();
      if (!item.closest('.dropdown-menu__item--dangerous')) return;
      // The confirmation's code loads a moment after the click.
      if (${deleteModal}) setTimeout(() => qaOpenModal(() => qaDeletes.push(id)), 60);
      else qaDeletes.push(id);
    };
  };
}
</script>`;

for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
  const { Agent } = await import(`../src/${build}/src/agent/agent.js`);
  test(`${build}: a Mastodon delete confirmation opened by WebBrain's own click keeps its post`, async t => {
    const browser = await engine.launch({ headless: true });
    try {
      async function fixture(options = {}) {
        const tab = await browser.newPage({ viewport: { width: 1126, height: 745 } });
        await tab.route('**/*', route => route.fulfill({ contentType: 'text/html', body: page(options) }));
        await tab.goto(`${origin}/home`);
        let scope = null;
        const runProbe = (tool, args) => tab.evaluate(({ source, tool, args }) => (0, eval)(`(${source})`)(tool, args),
          { source: Agent._submitActionProbe.toString(), tool, args });
        // A caption reading of null is a caption no model has read yet.
        const probe = (selector, read = fixtureControlEffectReading, overrides = {}) => probeWithReadings(runProbe, 'click', { selector,
          __wbNativeResourceAdapter: 'mastodon', __wbResourceRunToken: 'owned-run', __wbResourceTaskKey: 'owned-task',
          ...(scope ? { __wbOwnedResourceScope: scope } : {}), ...overrides }, read);
        // As the agent carries a delivered, non-submitting click's resource
        // to its next probe.
        const adopt = result => {
          if (result?.actionResource?.complete && result.isSubmit !== true) scope = { ...result.actionResource,
            document: result.frameDocumentIdentity, taskKey: 'owned-task', popupLauncherToken: result.popupLauncherToken || '',
            launcherDispatched: true };
        };
        // Playwright's click is trusted input; the trace's menu click was a
        // synthetic one.
        const click = async (selector, synthetic = false) => {
          if (synthetic) await tab.evaluate(selector => document.querySelector(selector).click(), selector);
          else await tab.click(selector);
          await tab.waitForTimeout(250);
        };
        const deletes = () => tab.evaluate(() => window.qaDeletes);
        return { tab, probe, adopt, click, deletes, get scope() { return scope; } };
      }
      const unread = () => null;

      await t.test('the trace: Sil in the menu not read yet, the confirmation it opens keeps the post', async () => {
        const f = await fixture();
        try {
          const more = await f.probe('#more-200');
          assert.equal(more.actionResource?.url, own('200')); assert.ok(more.popupLauncherToken);
          f.adopt(more); await f.click('#more-200');
          const item = await f.probe('#menu-delete', unread);
          assert.equal(item.isSubmit, false); assert.equal(item.resolvedPreparatoryTarget, true);
          assert.equal(item.mutationKind, 'delete', "Mastodon's dangerous entry of the user's own post deletes it");
          assert.equal(item.actionResource?.url, own('200')); assert.ok(item.popupLauncherToken);
          f.adopt(item); await f.click('#menu-delete', true);
          assert.deepEqual(await f.deletes(), [], 'The menu entry only opens the confirmation');
          const confirm = await f.probe('#confirm');
          assert.equal(confirm.mutationKind, 'delete'); assert.equal(confirm.semanticCommit, true);
          assert.equal(confirm.actionResource?.url, own('200'), 'The confirmation belongs to the post whose menu opened it');

          // The deletion guard then decides on that post, instead of refusing
          // the confirmation as unbound.
          const provider = { chat() {} }, agent = new Agent({ getActive: () => provider });
          agent.conversations.set(1, [{ role: 'system', content: 'system' }, { role: 'user', content: `Delete ${own('200')}.` }]);
          agent._startPlanExecutionGuard(1, 'act', { requestKind: 'execute', requiresStateChange: true, requiresSubmission: true });
          agent._detectLikelySubmitAction = async () => structuredClone(confirm);
          installModelStub(agent);
          assert.equal(await agent._resourceMutationPreSubmitBlock(1, 'click', { selector: '#confirm' }, confirm, provider), null);
          await f.click('#confirm');
          assert.deepEqual(await f.deletes(), ['200']);
        } finally { await f.tab.close(); }
      });

      for (const layout of ['narrow', 'wide']) await t.test(`${layout} window: the page made inert behind the confirmation still holds the post`, async () => {
        const f = await fixture({ layout });
        try {
          const more = await f.probe('#more-200'); f.adopt(more); await f.click('#more-200');
          const item = await f.probe('#menu-delete'); f.adopt(item); await f.click('#menu-delete');
          assert.equal(await f.tab.evaluate(() => document.querySelector('.columns-area__panels').inert), true);
          const confirm = await f.probe('#confirm');
          assert.equal(confirm.mutationKind, 'delete'); assert.equal(confirm.semanticCommit, true);
          assert.equal(confirm.actionResource?.url, own('200'));
          assert.equal(confirm.actionResource?.account, `${origin}/@ownedqa`);
        } finally { await f.tab.close(); }
      });

      // Where a browser lacks native inert, Mastodon's polyfill also stops
      // pointer events on the page behind the dialog.
      await t.test("an inert polyfill's pointer-events behind the confirmation hide nothing", async () => {
        const f = await fixture();
        try {
          await f.tab.addStyleTag({ content: '[inert]{pointer-events:none;cursor:default}' });
          const more = await f.probe('#more-200'); f.adopt(more); await f.click('#more-200');
          const item = await f.probe('#menu-delete'); f.adopt(item); await f.click('#menu-delete');
          assert.equal(await f.tab.evaluate(() => getComputedStyle(document.querySelector('#record-200')).pointerEvents), 'none');
          assert.equal((await f.probe('#confirm')).actionResource?.url, own('200'));
        } finally { await f.tab.close(); }
      });

      await t.test('no caption read at all: page structure alone binds and classifies the confirmation', async () => {
        const f = await fixture();
        try {
          const more = await f.probe('#more-200', unread); f.adopt(more); await f.click('#more-200');
          const item = await f.probe('#menu-delete', unread); f.adopt(item); await f.click('#menu-delete');
          const confirm = await f.probe('#confirm', unread);
          assert.equal(confirm.mutationKind, 'delete', "Mastodon's own confirm button is a deletion without any reading");
          assert.equal(confirm.semanticCommit, true); assert.equal(confirm.actionResource?.url, own('200'));
        } finally { await f.tab.close(); }
      });

      await t.test('Mastodon 4.6 and later: the confirmation form keeps the post', async () => {
        const f = await fixture({ shell: 'form' });
        try {
          const more = await f.probe('#more-201'); f.adopt(more); await f.click('#more-201');
          const item = await f.probe('#menu-delete'); f.adopt(item); await f.click('#menu-delete');
          const confirm = await f.probe('#confirm');
          assert.equal(confirm.mutationKind, 'delete'); assert.equal(confirm.actionResource?.url, own('201'));
          await f.click('#confirm');
          assert.deepEqual(await f.deletes(), ['201']);
        } finally { await f.tab.close(); }
      });

      // Mastodon 4.3 draws the menu's entries as links styled as buttons, and
      // below 1175 px shows no account bar at all.
      await t.test('Mastodon 4.3: link entries, no account bar, the confirmation still keeps the post', async () => {
        const f = await fixture({ entries: 'links', layout: 'hidden-account' });
        try {
          const more = await f.probe('#more-200'); f.adopt(more); await f.click('#more-200');
          assert.equal(await f.tab.evaluate(() => document.querySelector('#menu-delete').tagName), 'A');
          const item = await f.probe('#menu-delete', unread);
          assert.equal(item.mutationKind, 'delete'); assert.equal(item.actionResource?.url, own('200')); assert.ok(item.popupLauncherToken);
          f.adopt(item); await f.click('#menu-delete');
          const confirm = await f.probe('#confirm');
          assert.equal(confirm.mutationKind, 'delete'); assert.equal(confirm.actionResource?.url, own('200'));
        } finally { await f.tab.close(); }
      });

      // The probe identifies the page's posts once per run, however many of
      // its questions need them: a timeline of the user's own posts behind
      // the confirmation stays quick.
      await t.test('the posts behind the confirmation are identified once per probe run', async () => {
        const f = await fixture();
        try {
          const more = await f.probe('#more-200'); f.adopt(more); await f.click('#more-200');
          const item = await f.probe('#menu-delete'); f.adopt(item); await f.click('#menu-delete');
          await f.tab.evaluate(() => {
            window.qaRecordScans = 0;
            const scan = Document.prototype.querySelectorAll;
            Document.prototype.querySelectorAll = function (selector) {
              if (selector === '.status,.detailed-status') window.qaRecordScans++;
              return scan.call(this, selector);
            };
          });
          const readings = { Sil: 'delete_one', 'İptal': 'view' };
          const confirm = await f.probe('#confirm', fixtureControlEffectReading, { __wbCaptionPass: 'final', __wbCaptionReadings: readings });
          assert.equal(confirm.actionResource?.url, own('200'));
          assert.equal(await f.tab.evaluate(() => window.qaRecordScans), 1);
        } finally { await f.tab.close(); }
      });

      await t.test("the signed-in account comes from the app's own state where the navigation does not show it", async () => {
        const f = await fixture({ layout: 'hidden-account' });
        try {
          const more = await f.probe('#more-200'); f.adopt(more); await f.click('#more-200');
          assert.equal(more.actionResource?.url, own('200'));
          const item = await f.probe('#menu-delete'); f.adopt(item); await f.click('#menu-delete');
          assert.equal((await f.probe('#confirm')).actionResource?.url, own('200'));
        } finally { await f.tab.close(); }
        // Another signed-in account owns none of these posts, and an
        // initial state that disagrees with the navigation names no account.
        for (const options of [{ layout: 'hidden-account', signedIn: 'otherqa' }, { signedIn: 'otherqa' }]) {
          const other = await fixture(options);
          try { assert.equal((await other.probe('#more-200'))?.actionResource ?? null, null, JSON.stringify(options)); }
          finally { await other.tab.close(); }
        }
      });

      await t.test("another account's post binds nothing", async () => {
        const f = await fixture();
        try { assert.equal((await f.probe('#more-300'))?.actionResource ?? null, null); } finally { await f.tab.close(); }
      });

      await t.test('an inert page with no Mastodon dialog open still hides its posts', async () => {
        const f = await fixture();
        try {
          await f.tab.evaluate(() => document.querySelector('.columns-area__panels').setAttribute('inert', ''));
          assert.equal((await f.probe('#more-200'))?.actionResource ?? null, null);
          // A closed modal root does not excuse its inert siblings either.
          await f.tab.evaluate(() => { document.querySelector('.modal-root').innerHTML = '<div></div>'; });
          assert.equal((await f.probe('#more-200'))?.actionResource ?? null, null);
        } finally { await f.tab.close(); }
      });

      await t.test('a confirmation the click did not open borrows nothing', async () => {
        const f = await fixture();
        try {
          const more = await f.probe('#more-200'); f.adopt(more); await f.click('#more-200');
          const item = await f.probe('#menu-delete', unread); f.adopt(item);
          // Not clicked: the page opens the same confirmation by itself.
          await f.tab.evaluate(() => { document.querySelector('[id^="menu-"]').remove(); qaOpenModal(() => qaDeletes.push('script')); });
          await f.tab.waitForTimeout(250);
          assert.equal((await f.probe('#confirm'))?.actionResource ?? null, null);
        } finally { await f.tab.close(); }
      });

      await t.test('an ordinary menu entry is no deletion and lends its post to no confirmation', async () => {
        const f = await fixture();
        try {
          const more = await f.probe('#more-200'); f.adopt(more); await f.click('#more-200');
          const edit = await f.probe('#menu-edit', unread);
          assert.notEqual(edit?.mutationKind, 'delete'); assert.ok(!edit?.popupLauncherToken);
          f.adopt(edit); await f.click('#menu-edit');
          await f.tab.evaluate(() => qaOpenModal(() => qaDeletes.push('script')));
          await f.tab.waitForTimeout(250);
          assert.equal((await f.probe('#confirm'))?.actionResource ?? null, null);
        } finally { await f.tab.close(); }
      });

      // With Mastodon's confirmation preference off, its dangerous entry
      // deletes at once: it is checked as the deletion itself, read or not.
      await t.test('confirmation preference off: the dangerous entry is the deletion before any reading', async () => {
        const f = await fixture({ deleteModal: false });
        try {
          const more = await f.probe('#more-200'); f.adopt(more); await f.click('#more-200');
          const item = await f.probe('#menu-delete', unread);
          assert.equal(item.isSubmit, true); assert.equal(item.mutationKind, 'delete');
          assert.equal(item.actionResource?.url, own('200')); assert.equal(item.resolvedPreparatoryTarget, undefined);
        } finally { await f.tab.close(); }
      });
    } finally { await browser.close(); }
  });
}