Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Know GitHub's commit-dialog launcher and AWS WAF puzzle pages without English words
Claudec09b9790738d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b130f4948..90eedf62e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,8 @@ This changelog was generated from the repository Git history and release tags. V## [Unreleased]### Fixed+- An AWS WAF puzzle page ("Let's confirm you are human" in English) is now recognized in any language, by AWS's own puzzle script on its interstitial page, instead of by its English heading. Before, the same page in another language was not recognized as a CAPTCHA unless its widget was found, and a page running only AWS's silent check is still not taken for a puzzle.+- Editing a file on GitHub in another interface language: the "Commit changes…" button that only opens the commit dialog is now recognized by its closing ellipsis or by the dialog it names, in any language, instead of by the English words, so a Polish "Zatwierdź zmiany…" is treated like the English one. A commit message field beside the button is recognized by GitHub's own field id instead of its English label, so a button next to one keeps the full commit checks in every language.- Resolving GitHub review conversations: which buttons resolve a conversation is now read by a model in any language, instead of matched against their names in about ten languages. A Polish "Rozwiąż wątek" now counts as a conversation still to resolve. A page whose buttons no model could read yet is not taken for a pull request with nothing left to resolve.- Editing a video's details (YouTube Studio) and publishing posts, releases or content: which field of the saved page holds the title, description, visibility or another detail you asked for is now read by a model in any language, instead of being looked up in a list of field names in about 14 languages. A Polish "Tytuł (wymagany)" or "Opis" now counts as the title or description, so a correct save is recognized as finished, while a saved value that differs from the one you asked for, or two fields both read as the title, still is not. The labels are read when the task finishes, beside the page's status messages rather than after them, and only for fields read after the save.- File uploads in forms and release pages: whether the page shows your file as uploaded is now read by a model in any language, instead of looking for words such as "uploaded" or "eklendi". A Polish "cover.pdf · 96 KB · przesłano" now proves the upload, and a line saying the upload failed no longer counts as proof.diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex bab390b4a..3e852d8d9 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -2843,9 +2843,9 @@ export class Agent extends LoopDetector {...(Array.isArray(detectedSubmit?.fields) ? detectedSubmit.fields : []),...(Array.isArray(detectedSubmit?.changedFields) ? detectedSubmit.changedFields : []),];- const hasCommitMessageField = detectedFields.some(field => (- /\bcommit[\s_-]*message\b/i.test(String(field?.label || ''))- ));+ // The commit message fields are known by GitHub's own ids and names, in+ // any language of its interface.+ const hasCommitMessageField = detectedFields.some(field => this._isGithubCommitMessageField(field));// GitHub's first "Commit changes..." control only opens the commit dialog.// It can live inside the editor form and therefore look like a heuristic// submit, but the commit-message field is not present until the dialog is@@ -28300,9 +28300,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dtransactionOrderIdsComplete: formOrders.complete,transactionPageOrderIds: pageOrders.ids,transactionPageOrderIdsComplete: pageOrders.complete,+ // A caption that ends in an ellipsis opens a dialog to finish the+ // action, the way user interfaces mark it in every language.githubCommitDialogLauncher: githubEditPage&& !controlInModal- && (/^commit changes(?:\.{3}|…)?$/i.test(controlLabel) || controlOpensDialog),+ && (/(?:\.{3}|…)$/u.test(controlLabel) || controlOpensDialog),// The activated control's captions read as only searching,// filtering or showing something, in any language.searchSubmitCaption: captionsReading(controlCaptions(control)) === 'view',diff --git a/src/chrome/src/agent/captcha-frame-runtime.js b/src/chrome/src/agent/captcha-frame-runtime.jsindex 08e838635..73f2542ad 100644--- a/src/chrome/src/agent/captcha-frame-runtime.js+++ b/src/chrome/src/agent/captcha-frame-runtime.js@@ -1502,8 +1502,11 @@ export function observeAwsWafChallengeInPage() {const jsapiScript = awsScript('jsapi');const challengeScript = awsScript('challenge');const captchaScript = awsScript('captcha');- const fullPage = !!(goku?.key && (challengeScript || captchaScript)- && /let['’]s confirm you are human/i.test(document.body?.innerText || ''));+ // AWS WAF's own interstitial publishes gokuProps and loads its CAPTCHA+ // script only when it asks for a puzzle; a silent challenge loads the+ // challenge script alone and resolves by itself. That holds in every+ // language the interstitial is shown in, unlike its heading.+ const fullPage = !!(goku?.key && captchaScript);return {pageUrl: String(location.href),websiteKey: text(goku?.key),diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 3d604f911..390e06a38 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -2527,9 +2527,9 @@ export class Agent extends LoopDetector {...(Array.isArray(detectedSubmit?.fields) ? detectedSubmit.fields : []),...(Array.isArray(detectedSubmit?.changedFields) ? detectedSubmit.changedFields : []),];- const hasCommitMessageField = detectedFields.some(field => (- /\bcommit[\s_-]*message\b/i.test(String(field?.label || ''))- ));+ // The commit message fields are known by GitHub's own ids and names, in+ // any language of its interface.+ const hasCommitMessageField = detectedFields.some(field => this._isGithubCommitMessageField(field));// GitHub's first "Commit changes..." control only opens the commit dialog.// It can live inside the editor form and therefore look like a heuristic// submit, but the commit-message field is not present until the dialog is@@ -25641,9 +25641,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dtransactionOrderIdsComplete: formOrders.complete,transactionPageOrderIds: pageOrders.ids,transactionPageOrderIdsComplete: pageOrders.complete,+ // A caption that ends in an ellipsis opens a dialog to finish the+ // action, the way user interfaces mark it in every language.githubCommitDialogLauncher: githubEditPage&& !controlInModal- && (/^commit changes(?:\.{3}|…)?$/i.test(controlLabel) || controlOpensDialog),+ && (/(?:\.{3}|…)$/u.test(controlLabel) || controlOpensDialog),// The activated control's captions read as only searching,// filtering or showing something, in any language.searchSubmitCaption: captionsReading(controlCaptions(control)) === 'view',diff --git a/src/firefox/src/agent/captcha-frame-runtime.js b/src/firefox/src/agent/captcha-frame-runtime.jsindex 08e838635..73f2542ad 100644--- a/src/firefox/src/agent/captcha-frame-runtime.js+++ b/src/firefox/src/agent/captcha-frame-runtime.js@@ -1502,8 +1502,11 @@ export function observeAwsWafChallengeInPage() {const jsapiScript = awsScript('jsapi');const challengeScript = awsScript('challenge');const captchaScript = awsScript('captcha');- const fullPage = !!(goku?.key && (challengeScript || captchaScript)- && /let['’]s confirm you are human/i.test(document.body?.innerText || ''));+ // AWS WAF's own interstitial publishes gokuProps and loads its CAPTCHA+ // script only when it asks for a puzzle; a silent challenge loads the+ // challenge script alone and resolves by itself. That holds in every+ // language the interstitial is shown in, unlike its heading.+ const fullPage = !!(goku?.key && captchaScript);return {pageUrl: String(location.href),websiteKey: text(goku?.key),diff --git a/test/captcha-native-providers.mjs b/test/captcha-native-providers.mjsindex e498e3b57..e51c124ae 100644--- a/test/captcha-native-providers.mjs+++ b/test/captcha-native-providers.mjs@@ -83,6 +83,23 @@ for (const browser of ['chrome','firefox']) {assert.equal(native.describeAwsWafObservation(providers, { pageUrl: observed.pageUrl }), null);});+ test(`${browser}: an AWS WAF puzzle page is known by its own script in any language, not by its English heading`, async () => {+ const runtime = await import(`../src/${browser}/src/agent/captcha-frame-runtime.js`);+ const observe = ({ goku = true, scripts }) => {+ const saved = { window: globalThis.window, document: globalThis.document, location: globalThis.location };+ globalThis.window = goku ? { gokuProps: { key: 'AQIDAHjc', iv: 'CgAH', context: 'ctx' } } : {};+ globalThis.document = { scripts: scripts.map(src => ({ src })), querySelector: () => null,+ body: { innerText: 'Potwierdźmy, że jesteś człowiekiem\nRozpocznij' } };+ globalThis.location = { href: 'https://site.test/join' };+ try { return runtime.observeAwsWafChallengeInPage(); } finally { Object.assign(globalThis, saved); }+ };+ const base = 'https://abc.edge.token.awswaf.com/abc/def';+ assert.equal(observe({ scripts: [`${base}/challenge.js`, `${base}/captcha.js`] }).active, true, 'a Polish puzzle page');+ assert.equal(observe({ scripts: [`${base}/challenge.js`] }).active, false, 'a silent challenge resolves by itself');+ assert.equal(observe({ goku: false, scripts: [`${base}/captcha.js`] }).active, false, 'a site\'s own CAPTCHA script with no interstitial');+ assert.equal(observe({ scripts: ['https://evil.test/awswaf.com/captcha.js'] }).active, false, 'a script off awswaf.com');+ });+test(`${browser}: every CapMonster recognition model uses the documented wire discriminator`,()=>{const names={bills_audio:'bills_audio',shein:'shein',bls:'bls_3x3',baidu:'baidu',betpunch_3x3_rotate:'betpunch_3x3_rotate',oocl_rotate_double_new:'oocl_rotate_double_new',oocl_rotate_new:'oocl_rotate_new',dli_ensemble:'dli',mathsum:'MathSum',portugal_text_find_icon:'portugal_text_find_icon'};for(const [id,Task] of Object.entries(names)) {diff --git a/test/run.js b/test/run.jsindex 2865abca0..dfd082870 100644--- a/test/run.js+++ b/test/run.js@@ -82528,6 +82528,20 @@ test('GitHub edit-file workflow verifies the exact committed raw blob', async ()fields: [{ label: 'Editing file contents' }],},), null, `${label}: the reversible commit-dialog launcher was blocked before its message field existed`);+ // A commit message field beside the control, known by GitHub's own id in+ // any interface language, means the click commits rather than opening+ // the dialog.+ assert.equal((await agent._workflowPreSubmitDispatchBlock(+ tabId,+ 'click_ax',+ {},+ {+ isSubmit: true,+ validationSubmitEvidence: 'heuristic',+ githubCommitDialogLauncher: true,+ fields: [{ label: 'Editing file contents' }, { label: 'Wiadomość zatwierdzenia', id: 'commit-message-input' }],+ },+ ))?.noDispatch, true, `${label}: a launcher beside a commit message field was taken for the reversible dialog launcher`);assert.equal((await agent._workflowPreSubmitDispatchBlock(tabId,'click_ax',@@ -85039,7 +85053,7 @@ test('submit detector source covers submit controls, Enter, set_field, iframes,assert.match(agent, /publicationAccountIdentityComplete: publicationAccount\.complete/, `${label}: account evidence completeness should survive the page probe`);assert.match(agent, /evidence\.strong \? 'strong' : 'heuristic'/, `${label}: custom submit probes should label strong and heuristic evidence`);assert.match(agent, /detected\.validationSubmitEvidence === 'strong' \? 'strong' : 'heuristic'/, `${label}: submit evidence strength should survive page-probe normalization`);- assert.match(agent, /githubCommitDialogLauncher: githubEditPage[\s\S]*!controlInModal[\s\S]*commit changes/, `${label}: GitHub's reversible commit-dialog launcher must be target- and modal-bound`);+ assert.match(agent, /githubCommitDialogLauncher: githubEditPage\s*&& !controlInModal\s*&& \(\/\(\?:\\\.\{3\}\|…\)\$\/u\.test\(controlLabel\) \|\| controlOpensDialog\)/, `${label}: GitHub's reversible commit-dialog launcher must be target- and modal-bound`);assert.match(agent, /const findTopmostModal = \(\) => \{[\s\S]*dialog\[open\][\s\S]*\[role="dialog"\]\[aria-modal="true"\][\s\S]*\[class\*="DialogOverlay"\]/, `${label}: text submit probing should mirror modal scoping`);assert.match(agent, /Array\.from\(\(findTopmostModal\(\) \|\| doc\)\.querySelectorAll/, `${label}: text submit probing should search inside the topmost modal when present`);}diff --git a/test/submit-probe-readings.mjs b/test/submit-probe-readings.mjsindex 616e1ab49..eb280a377 100644--- a/test/submit-probe-readings.mjs+++ b/test/submit-probe-readings.mjs@@ -134,6 +134,27 @@ for (const build of ['chrome', 'firefox']) {assert.equal(plain.searchLandmark, false);assert.equal(plain.searchSubmitCaption, false);});++ await t.test('a GitHub commit-dialog launcher is known by its ellipsis or dialog trigger, in any language', async () => {+ await page.goto('https://github.com/owned/repo/edit/main/README.md');+ // A caption ending in an ellipsis opens the commit dialog, whatever+ // its language; so does a control that names the dialog it opens.+ await open('<form><button id="t">Zatwierdź zmiany…</button></form>');+ assert.equal((await probe('#t')).githubCommitDialogLauncher, true, 'a Polish launcher with an ellipsis');+ await open('<form><button id="t">Commit changes...</button></form>');+ assert.equal((await probe('#t')).githubCommitDialogLauncher, true, 'an English launcher with three dots');+ await open('<form><button id="t" data-show-dialog-id="commit-dialog">Zatwierdź</button></form>');+ assert.equal((await probe('#t')).githubCommitDialogLauncher, true, 'a launcher naming its dialog');+ // A caption with no ellipsis and no dialog commits, as does the+ // dialog's own button and any button off the edit page.+ await open('<form><button id="t">Zatwierdź zmiany</button></form>');+ assert.equal((await probe('#t')).githubCommitDialogLauncher, false, 'a commit button with no ellipsis');+ await open('<div role="dialog" aria-modal="true"><form><button id="t">Zatwierdź zmiany…</button></form></div>');+ assert.equal((await probe('#t')).githubCommitDialogLauncher, false, 'a button inside the commit dialog');+ await page.goto(pageUrl);+ await open('<form><button id="t">Zatwierdź zmiany…</button></form>');+ assert.equal((await probe('#t')).githubCommitDialogLauncher, false, 'a button off the GitHub edit page');+ });} finally {await browser.close();}