Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Read a wall in front of an article by meaning, in any language
Claudea5af906cc9a6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c0fb30ac8..1d64606dd 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,7 @@ This changelog was generated from the repository Git history and release tags. V## [Unreleased]### Fixed+- A subscription, registration or sign-in wall in front of a news article is now recognized in any language, by a model reading the wall's text once, instead of only when it used English phrases such as "Subscribe to continue reading". A Turkish "Okumaya devam etmek için abone olun" overlay now keeps WebBrain from reading the article hidden behind it, as an English one did, while an offer to subscribe beside text you can still read is not taken for a wall.- A dialog asking you to prove you are human now pauses the run's page actions in any language, not only when it says "Security verification", "Verify you are human" or another English phrase. Its title is read by a model once, the decision model first; LinkedIn's Turkish "Güvenlik doğrulaması" now counts, while a cookie notice or a form dialog does not. A dialog naming a CAPTCHA product still counts without any reading.- Finding a verification email: the service the code is for is now matched by its name or web address alone, with no English words such as "code" or "verification" set aside, so the search treats the same request in every language alike. The tool now asks the model for only the service's name or domain.- An AWS WAF puzzle page ("Let's confirm you are human" in English) is now recognized in any language, by AWS's own puzzle script on its interstitial page, instead of by its English heading. Before, the same page in another language was not recognized as a CAPTCHA unless its widget was found, and a page running only AWS's silent check is still not taken for a puzzle.diff --git a/docs/architecture.md b/docs/architecture.mdindex 8a533d978..3403345d3 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -576,6 +576,16 @@ preactivates enabled FreeSkillz for that run; only a structured blocking`pageGate` adds the trusted instruction to call `fetch_nytimes_article`, so rawpage prose cannot spoof fallback routing.+A `pageGate` is a wall in front of an article: an overlay on an article page, or+a block inside the article, whose text a model read as keeping the article+from the reader until they subscribe, register or sign in (the `page_blocker`+question, in any language; an offer beside readable text is not a wall). The+page read reports an overlay or block text nobody has read in+`pageTextsNeeded`; the agent has it read (the decision model first, the active+model within four seconds) and reads the page once more with the reading. An+unread text is no wall, so the page reads as it is. A wall withholds the+article, its links, forms and frames; its own text and controls stay readable.+Single public-media downloads have a second deterministic guard. If the modelcalls `download_social_media` while an eligible inactive skill owns`download_public_media`, the runtime activates that skill and returns a retrydiff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex f87b5c8b2..83ec2f70f 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -12110,6 +12110,52 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn outcome;}+ // The page_blocker readings the agent holds, sent with a page read so the+ // content script's page-gate check reads overlay and article-block texts by+ // meaning (see detectPageGate in content.js).+ _withPageBlockerReadings(name, params) {+ if (name !== 'read_page' && name !== 'get_accessibility_tree') return params;+ return { ...(params && typeof params === 'object' ? params : {}),+ __wbPageReadings: { page_blocker: this._challengeTitleReadings() } };+ }++ // Overlay and article-block texts a page read consulted unread, read in any+ // language (the decision model first, the active model within four+ // seconds). True when any of them was read.+ async _readPageBlockerTexts(tabId, entries) {+ const texts = [...new Set((Array.isArray(entries) ? entries : [])+ .filter(entry => entry?.kind === 'page_blocker')+ .map(entry => String(entry?.text ?? '').replace(/\s+/g, ' ').trim())+ .filter(text => text && text.length <= 400 && !this._pageTextReading('page_blocker', text)))].slice(0, 12);+ if (!texts.length) return false;+ try {+ await this._readPageTexts(tabId, 'page_blocker', texts, {}, { maxChars: 400, mainTimeoutMs: CONTROL_READING_TIMEOUT_MS });+ } catch (error) {+ if (this._checkAbort(tabId)) throw error;+ }+ return texts.some(text => !!this._pageTextReading('page_blocker', text));+ }++ // A page read whose overlay or article-block texts were consulted unread+ // has them read, and the page read once more with the readings; a read+ // that cannot be repeated keeps the first answer.+ async _rereadWithPageBlockerReadings(tabId, name, response, dispatch) {+ if ((name !== 'read_page' && name !== 'get_accessibility_tree') || !response || typeof response !== 'object') return response;+ const needed = response.pageTextsNeeded;+ delete response.pageTextsNeeded;+ if (!await this._readPageBlockerTexts(tabId, needed)) return response;+ let reread = null;+ try {+ reread = await dispatch();+ } catch (error) {+ if (error?.name === 'AbortError' || this._checkAbort(tabId)) throw error;+ return response;+ }+ if (!reread || typeof reread !== 'object' || reread.success === false) return response;+ delete reread.pageTextsNeeded;+ return reread;+ }+// The page_blocker readings of dialog titles the agent holds, by title, for// the CAPTCHA scans that run in the page._challengeTitleReadings() {@@ -42134,7 +42180,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (name === 'get_shadow_dom') {try {await cdpClient.attach(tabId);- const pageInfo = await cdpClient.readPage(tabId);+ // Shadow hosts behind a wall in front of an article stay hidden; the+ // wall's text is read in any language first when nobody has.+ let pageInfo = await cdpClient.readPage(tabId, { pageBlockerReadings: this._challengeTitleReadings() });+ if (await this._readPageBlockerTexts(tabId, pageInfo?.pageTextsNeeded)) {+ pageInfo = await cdpClient.readPage(tabId, { pageBlockerReadings: this._challengeTitleReadings() });+ }return {success: true,shadowHosts: pageInfo.shadowHosts || [],@@ -45594,7 +45645,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn chrome.tabs.sendMessage(tabId, {target: 'content',action,- params: contentArgs,+ params: this._withPageBlockerReadings(name, contentArgs),...(actionDeadlineAt > 0 ? { actionDeadlineAt } : {}),}, messageOptions);};@@ -45715,6 +45766,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d);}}+ response = await this._rereadWithPageBlockerReadings(tabId, name, response, dispatchContentAction);if (name === 'click') {response = await this._settleContentFilePickerGuard(tabId, response);}diff --git a/src/chrome/src/agent/semantic-decisions.js b/src/chrome/src/agent/semantic-decisions.jsindex 34588a6d3..d4c62b9c9 100644--- a/src/chrome/src/agent/semantic-decisions.js+++ b/src/chrome/src/agent/semantic-decisions.js@@ -379,11 +379,13 @@ export const SEMANTIC_QUESTIONS = Object.freeze({choices: Object.freeze({human_check: 'It asks them to prove they are a person and not a robot: a CAPTCHA, a puzzle, a robot check, or a '+ 'security or human verification.',- subscription: 'It asks them to subscribe, pay or start a trial to read the page\'s article or content.',- registration: 'It asks them to create an account or register to read the page\'s article or content.',- login: 'It asks them to sign in to read the page\'s article or content.',- other: 'Something else, such as a cookie or privacy notice, a newsletter offer, an advertisement, a form to fill '- + 'in, a confirmation, or a message they can read past.',+ subscription: 'It is a wall that keeps the page\'s article or content from them until they subscribe, pay or '+ + 'start a trial, such as "Subscribe to continue reading".',+ registration: 'It is a wall that keeps the page\'s article or content from them until they create an account or '+ + 'register.',+ login: 'It is a wall that keeps the page\'s article or content from them until they sign in.',+ other: 'Something else, such as a cookie or privacy notice, an offer to subscribe or a newsletter beside content '+ + 'they can still read, an advertisement, a form to fill in, a confirmation, or a message they can read past.',}),// Each answer but `other` stops actions or limits reading, so it needs a// sure reading; a title nobody could read stops nothing.diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.jsindex 86d565184..bfda5e083 100644--- a/src/chrome/src/cdp/cdp-client.js+++ b/src/chrome/src/cdp/cdp-client.js@@ -3964,10 +3964,16 @@ export class CDPClient {* Pass `{ includeChrome: true }` to opt out of stripping (e.g. when the* user is asking about the nav, footer, cookie banner, etc.).*/+ // `opts.pageBlockerReadings` are the agent's page_blocker readings by text:+ // an overlay or article block counts as a wall only once its text was read+ // as one, in any language; texts nobody has read come back in+ // `pageTextsNeeded` (see detectPageGate in content.js).async readPage(tabId, opts = {}) {const includeChrome = !!opts.includeChrome;+ const pageBlockerReadings = opts.pageBlockerReadings && typeof opts.pageBlockerReadings === 'object'+ ? opts.pageBlockerReadings : {};const pageInfo = await this.evaluate(tabId, `- ((includeChrome) => {+ ((includeChrome, pageBlockerReadings) => {const ARTICLE_SELECTORS = ['[itemprop="articleBody"]','article [class*="article-body" i]',@@ -4021,8 +4027,6 @@ export class CDPClient {return rect.width >= 20 && rect.height >= 20;} catch (e) { return false; }};- const gateHasAccessLanguage = (text) => /(?:(?:subscribe|subscription).{0,48}(?:continue|read|access|article|options|required|unlock)|subscriber[- ]only|unlimited access|unlock (?:this|the) article|start (?:your )?(?:free )?trial|create (?:a )?(?:free )?account|register to (?:continue|read)|sign up to (?:continue|read)|(?:log|sign) in to (?:continue|read)|continue reading (?:with|by)|to continue reading|already have an account)/.test(String(text || '').replace(/\\s+/g, ' ').trim().toLowerCase());- const gateHasInlineBlockingLanguage = (text) => /(?:to continue reading|continue reading (?:with|by)|subscriber[- ]only|(?:subscribe|subscription required|register|sign up|log in|sign in|create (?:a )?(?:free )?account).{0,64}(?:continue reading|read (?:this |the )?(?:full )?(?:article|story)|unlock (?:this|the) (?:article|story)|access (?:this|the) (?:article|story)))/.test(String(text || '').replace(/\\s+/g, ' ').trim().toLowerCase());const pageHasArticleContext = () => {try {return !!(@@ -4033,37 +4037,16 @@ export class CDPClient {);} catch (e) { return false; }};- const boundedGateLabel = (el, rawLabel) => {- const options = [];- let boundaryElement = null;- for (const value of [el.getAttribute('aria-label'), el.getAttribute('title')]) {- const normalized = String(value || '').replace(/\\s+/g, ' ').trim();- if (gateHasAccessLanguage(normalized)) options.push(normalized);- }- let descendants = [];- try { descendants = el.querySelectorAll('h1, h2, h3, p, button, a, label, [role="heading"]'); } catch (e) {}- for (const node of Array.from(descendants).slice(0, 80)) {- if (!gateElementIsRendered(node)) continue;- const normalized = String(node.innerText || '').replace(/\\s+/g, ' ').trim();- if (normalized.length <= 600 && gateHasAccessLanguage(normalized)) {- boundaryElement ||= node;- options.push(normalized);- }+ const unreadGateTexts = [];+ const gateText = (el) => String(el?.innerText || '').replace(/[\\u200b-\\u200d\\ufeff]/g, '').replace(/\\s+/g, ' ').trim().slice(0, 400);+ const gateReading = (text) => {+ if (!text) return null;+ const reading = Object.prototype.hasOwnProperty.call(pageBlockerReadings, text) ? String(pageBlockerReadings[text] || '') : '';+ if (!reading) {+ if (!unreadGateTexts.includes(text) && unreadGateTexts.length < 12) unreadGateTexts.push(text);+ return null;}- options.sort((a, b) => a.length - b.length);- if (options[0]) return { label: options[0].slice(0, 240), boundaryElement };- const accessStart = rawLabel.search(/\\b(?:subscribe|subscription|subscriber|unlock|register|sign up|log in|sign in|create an? account|continue reading)\\b/i);- return {- label: rawLabel.slice(Math.max(0, accessStart), Math.max(0, accessStart) + 240),- boundaryElement,- };- };- const gateType = (text) => {- const value = String(text || '').toLowerCase();- if (/\\bsubscribe\\b|\\bsubscription\\b|subscriber[- ]only|unlimited access|unlock (?:this|the) article|start (?:your )?(?:free )?trial/.test(value)) return 'subscription';- if (/create (?:a )?(?:free )?account|register to (?:continue|read)|sign up to (?:continue|read)/.test(value)) return 'registration';- if (/(?:log|sign) in to (?:continue|read)|already have an account|create (?:a )?(?:free )?account or log in/.test(value)) return 'login';- return 'unknown';+ return ['subscription', 'registration', 'login'].includes(reading) ? reading : null;};const detectPageGate = () => {const seen = new Set();@@ -4075,7 +4058,6 @@ export class CDPClient {for (const el of matches) {if (seen.has(el) || !gateElementIsRendered(el)) continue;seen.add(el);- const rawLabel = String(el.innerText || '').replace(/\\s+/g, ' ').trim();const role = String(el.getAttribute('role') || '').toLowerCase();let surface = (role === 'dialog' || role === 'alertdialog' || el.tagName === 'DIALOG' || el.getAttribute('aria-modal') === 'true') ? 'dialog' : 'inline';const inArticle = !!el.closest('article, [role="article"], main, [role="main"]');@@ -4089,16 +4071,13 @@ export class CDPClient {coveringOverlay = ['fixed', 'sticky', 'absolute'].includes(style.position) && ((visibleWidth * visibleHeight) / viewportArea) >= 0.2;} catch (e) {}if (coveringOverlay) surface = 'dialog';- if (surface === 'dialog') {- if (!articleContext || !gateHasAccessLanguage(rawLabel)) continue;- } else {- if (!inArticle || !gateHasInlineBlockingLanguage(rawLabel)) continue;- }- const gateText = boundedGateLabel(el, rawLabel);- const label = gateText.label;+ if (surface === 'dialog' ? !articleContext : !inArticle) continue;+ const label = gateText(el);+ const type = gateReading(label);+ if (!type) continue;const namedGate = /paywall|gateway|regiwall|subscription|registration/i.test([el.id, typeof el.className === 'string' ? el.className : '', el.getAttribute('data-testid') || ''].join(' '));const score = (surface === 'dialog' ? 100 : 0) + (inArticle ? 40 : 0) + (coveringOverlay ? 30 : 0) + (namedGate ? 15 : 0) - Math.min(label.length, 2000) / 2000;- candidates.push({ element: surface === 'inline' ? (gateText.boundaryElement || el) : el, type: gateType(rawLabel), surface, label: label.slice(0, 240), score });+ candidates.push({ element: el, type, surface, label: label.slice(0, 240), score });}}candidates.sort((a, b) => b.score - a.score);@@ -4220,6 +4199,7 @@ export class CDPClient {title: document.title,description: document.querySelector('meta[name="description"]')?.content || '',...(pageGate ? { pageGate } : {}),+ ...(unreadGateTexts.length ? { pageTextsNeeded: unreadGateTexts.map(text => ({ kind: 'page_blocker', text })) } : {}),text,textSource,isArticlePage,@@ -4252,7 +4232,7 @@ export class CDPClient {visible: iframe.offsetWidth > 0 && iframe.offsetHeight > 0,})),};- })(${JSON.stringify(includeChrome)})+ })(${JSON.stringify(includeChrome)}, ${JSON.stringify(pageBlockerReadings)})`);return pageInfo?.result?.value || pageInfo;diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 7fdefb922..dd5d3ee32 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -3519,30 +3519,21 @@return rect.width >= 20 && rect.height >= 20;}- function pageGateType(text) {- const value = String(text || '').toLowerCase();- if (/\bsubscribe\b|\bsubscription\b|subscriber[- ]only|unlimited access|unlock (?:this|the) article|start (?:your )?(?:free )?trial/.test(value)) {- return 'subscription';- }- if (/create (?:a )?(?:free )?account|register to (?:continue|read)|sign up to (?:continue|read)/.test(value)) {- return 'registration';- }- if (/(?:log|sign) in to (?:continue|read)|already have an account|create (?:a )?(?:free )?account or log in/.test(value)) {- return 'login';- }- return 'unknown';- }+ // What an overlay or a block in an article asks of its reader, read by a+ // model in any language from its text (the agent's page_blocker readings):+ // a subscription, registration or sign-in wall in front of the article. A+ // text nobody has read yet is no wall; it is reported for the agent to+ // read, and the agent asks again with the reading.+ const PAGE_GATE_TYPES = new Set(['subscription', 'registration', 'login']);- function pageGateHasAccessLanguage(text) {- const value = String(text || '').replace(/\s+/g, ' ').trim().toLowerCase();- if (!value) return false;- return /(?:(?:subscribe|subscription).{0,48}(?:continue|read|access|article|options|required|unlock)|subscriber[- ]only|unlimited access|unlock (?:this|the) article|start (?:your )?(?:free )?trial|create (?:a )?(?:free )?account|register to (?:continue|read)|sign up to (?:continue|read)|(?:log|sign) in to (?:continue|read)|continue reading (?:with|by)|to continue reading|already have an account)/.test(value);+ function pageGateText(el) {+ return _pageReadingText(el?.innerText || '').slice(0, 400);}- function pageGateHasInlineBlockingLanguage(text) {- const value = String(text || '').replace(/\s+/g, ' ').trim().toLowerCase();- if (!value) return false;- return /(?:to continue reading|continue reading (?:with|by)|subscriber[- ]only|(?:subscribe|subscription required|register|sign up|log in|sign in|create (?:a )?(?:free )?account).{0,64}(?:continue reading|read (?:this |the )?(?:full )?(?:article|story)|unlock (?:this|the) (?:article|story)|access (?:this|the) (?:article|story)))/.test(value);+ function pageGateReading(text) {+ if (!text) return null;+ const reading = _pageReading('page_blocker', text, null);+ return PAGE_GATE_TYPES.has(reading) ? reading : null;}function pageHasArticleContext() {@@ -3558,34 +3549,6 @@}}- function boundedPageGateLabel(el, rawLabel) {- const options = [];- let boundaryElement = null;- for (const value of [el.getAttribute('aria-label'), el.getAttribute('title')]) {- const normalized = String(value || '').replace(/\s+/g, ' ').trim();- if (pageGateHasAccessLanguage(normalized)) options.push(normalized);- }- let descendants = [];- try {- descendants = el.querySelectorAll('h1, h2, h3, p, button, a, label, [role="heading"]');- } catch { /* use the bounded surface text below */ }- for (const node of Array.from(descendants).slice(0, 80)) {- if (!gateElementIsRendered(node)) continue;- const normalized = String(node.innerText || '').replace(/\s+/g, ' ').trim();- if (normalized.length <= 600 && pageGateHasAccessLanguage(normalized)) {- boundaryElement ||= node;- options.push(normalized);- }- }- options.sort((a, b) => a.length - b.length);- if (options[0]) return { label: options[0].slice(0, 240), boundaryElement };- const accessStart = rawLabel.search(/\b(?:subscribe|subscription|subscriber|unlock|register|sign up|log in|sign in|create an? account|continue reading)\b/i);- return {- label: rawLabel.slice(Math.max(0, accessStart), Math.max(0, accessStart) + 240),- boundaryElement,- };- }-function pageGatePublic(gate) {if (!gate) return null;return {@@ -3606,13 +3569,9 @@for (const el of matches) {if (seen.has(el) || !gateElementIsRendered(el)) continue;seen.add(el);- const rawLabel = String(el.innerText || '').replace(/\s+/g, ' ').trim();-const role = String(el.getAttribute('role') || '').toLowerCase();- let surface = (- role === 'dialog' || role === 'alertdialog' ||- el.tagName === 'DIALOG' || el.getAttribute('aria-modal') === 'true'- ) ? 'dialog' : 'inline';+ let surface = (role === 'dialog' || role === 'alertdialog' || el.tagName === 'DIALOG'+ || el.getAttribute('aria-modal') === 'true') ? 'dialog' : 'inline';const inArticle = !!el.closest('article, [role="article"], main, [role="main"]');let coveringOverlay = false;try {@@ -3625,28 +3584,20 @@((visibleWidth * visibleHeight) / viewportArea) >= 0.2;} catch { /* rendered check already supplied safe defaults */ }if (coveringOverlay) surface = 'dialog';- if (surface === 'dialog') {- if (!articleContext || !pageGateHasAccessLanguage(rawLabel)) continue;- } else {- if (!inArticle || !pageGateHasInlineBlockingLanguage(rawLabel)) continue;- }- const gateText = boundedPageGateLabel(el, rawLabel);- const label = gateText.label;+ // An overlay counts only on an article page, and a block only inside+ // the article; its text then has to read as a wall.+ if (surface === 'dialog' ? !articleContext : !inArticle) continue;+ const label = pageGateText(el);+ const type = pageGateReading(label);+ if (!type) continue;const namedGate = /paywall|gateway|regiwall|subscription|registration/i.test([el.id,typeof el.className === 'string' ? el.className : '',el.getAttribute('data-testid') || '',].join(' '));-const score = (surface === 'dialog' ? 100 : 0) + (inArticle ? 40 : 0) +(coveringOverlay ? 30 : 0) + (namedGate ? 15 : 0) - Math.min(label.length, 2000) / 2000;- candidates.push({- element: surface === 'inline' ? (gateText.boundaryElement || el) : el,- type: pageGateType(rawLabel),- surface,- label: label.slice(0, 240),- score,- });+ candidates.push({ element: el, type, surface, label: label.slice(0, 240), score });}}candidates.sort((a, b) => b.score - a.score);@@ -6475,6 +6426,7 @@message_field: new Set(['to', 'cc', 'bcc', 'subject', 'body', 'other']),message_author: new Set(['self', 'other']),message_status: new Set(['delivered', 'sent', 'pending', 'unclear']),+ page_blocker: new Set(['human_check', 'subscription', 'registration', 'login', 'other']),});// Readings that name an action of the control itself, never a contact's name.const _MESSAGE_CONTROL_ACTIONS = new Set(['send', 'publish', 'reply', 'new_message', 'new_post',@@ -6532,9 +6484,15 @@return reading === 'send' || reading === null;}- function _takeUnreadPageTexts() {- const texts = [..._pageReadingsUnread.values()];- _pageReadingsUnread.clear();+ // The texts consulted unread, all of them or those of one question, taken+ // from the queue for the agent to read.+ function _takeUnreadPageTexts(kind = '') {+ const texts = [];+ for (const [key, entry] of [..._pageReadingsUnread.entries()]) {+ if (kind && entry.kind !== kind) continue;+ texts.push(entry);+ _pageReadingsUnread.delete(key);+ }return texts;}@@ -10372,6 +10330,15 @@if (!value.refScopeUrl) value.refScopeUrl = location.href;} catch { /* identity is best-effort; never break the response */ }}+ // Overlay and article-block texts the page-gate check consulted unread+ // go back to the agent to be read; it asks again with the readings.+ if (['get_page_info', 'get_page_info_cdp', 'get_accessibility_tree'].includes(msg.action)+ && value && typeof value === 'object') {+ const unread = _takeUnreadPageTexts('page_blocker');+ if (unread.length) {+ value.pageTextsNeeded = [...(Array.isArray(value.pageTextsNeeded) ? value.pageTextsNeeded : []), ...unread];+ }+ }return value;};diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 57b77f48b..282b98f69 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -10749,6 +10749,52 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d};}+ // The page_blocker readings the agent holds, sent with a page read so the+ // content script's page-gate check reads overlay and article-block texts by+ // meaning (see detectPageGate in content.js).+ _withPageBlockerReadings(name, params) {+ if (name !== 'read_page' && name !== 'get_accessibility_tree') return params;+ return { ...(params && typeof params === 'object' ? params : {}),+ __wbPageReadings: { page_blocker: this._challengeTitleReadings() } };+ }++ // Overlay and article-block texts a page read consulted unread, read in any+ // language (the decision model first, the active model within four+ // seconds). True when any of them was read.+ async _readPageBlockerTexts(tabId, entries) {+ const texts = [...new Set((Array.isArray(entries) ? entries : [])+ .filter(entry => entry?.kind === 'page_blocker')+ .map(entry => String(entry?.text ?? '').replace(/\s+/g, ' ').trim())+ .filter(text => text && text.length <= 400 && !this._pageTextReading('page_blocker', text)))].slice(0, 12);+ if (!texts.length) return false;+ try {+ await this._readPageTexts(tabId, 'page_blocker', texts, {}, { maxChars: 400, mainTimeoutMs: CONTROL_READING_TIMEOUT_MS });+ } catch (error) {+ if (this._checkAbort(tabId)) throw error;+ }+ return texts.some(text => !!this._pageTextReading('page_blocker', text));+ }++ // A page read whose overlay or article-block texts were consulted unread+ // has them read, and the page read once more with the readings; a read+ // that cannot be repeated keeps the first answer.+ async _rereadWithPageBlockerReadings(tabId, name, response, dispatch) {+ if ((name !== 'read_page' && name !== 'get_accessibility_tree') || !response || typeof response !== 'object') return response;+ const needed = response.pageTextsNeeded;+ delete response.pageTextsNeeded;+ if (!await this._readPageBlockerTexts(tabId, needed)) return response;+ let reread = null;+ try {+ reread = await dispatch();+ } catch (error) {+ if (error?.name === 'AbortError' || this._checkAbort(tabId)) throw error;+ return response;+ }+ if (!reread || typeof reread !== 'object' || reread.success === false) return response;+ delete reread.pageTextsNeeded;+ return reread;+ }+// The page_blocker readings of dialog titles the agent holds, by title, for// the CAPTCHA scans that run in the page._challengeTitleReadings() {@@ -39608,7 +39654,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst response = await firefoxBidi.sendContent(tabId, {target: 'content',action,- params: contentArgs,+ params: this._withPageBlockerReadings(name, contentArgs),...(actionDeadlineAt > 0 ? { actionDeadlineAt } : {}),}, messageOptions, { onDispatch: Agent.STATE_CHANGE_TOOLS.has(name) ? () => {throwIfContentPipelineAborted();@@ -39660,6 +39706,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}const deadlineResult = provenNoDispatchDeadline(response);if (deadlineResult) return this._withCoordinateReconciliation(deadlineResult, coordinateDiagnostic);+ response = await this._rereadWithPageBlockerReadings(tabId, name, response, dispatchContentAction);if (name === 'click') {response = await this._settleContentFilePickerGuard(tabId, response);}@@ -39725,6 +39772,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dlet response = await dispatchContentAction();const deadlineResult = provenNoDispatchDeadline(response);if (deadlineResult) return this._withCoordinateReconciliation(deadlineResult, coordinateDiagnostic);+ response = await this._rereadWithPageBlockerReadings(tabId, name, response, dispatchContentAction);if (name === 'click') {response = await this._settleContentFilePickerGuard(tabId, response);}diff --git a/src/firefox/src/agent/semantic-decisions.js b/src/firefox/src/agent/semantic-decisions.jsindex 34588a6d3..d4c62b9c9 100644--- a/src/firefox/src/agent/semantic-decisions.js+++ b/src/firefox/src/agent/semantic-decisions.js@@ -379,11 +379,13 @@ export const SEMANTIC_QUESTIONS = Object.freeze({choices: Object.freeze({human_check: 'It asks them to prove they are a person and not a robot: a CAPTCHA, a puzzle, a robot check, or a '+ 'security or human verification.',- subscription: 'It asks them to subscribe, pay or start a trial to read the page\'s article or content.',- registration: 'It asks them to create an account or register to read the page\'s article or content.',- login: 'It asks them to sign in to read the page\'s article or content.',- other: 'Something else, such as a cookie or privacy notice, a newsletter offer, an advertisement, a form to fill '- + 'in, a confirmation, or a message they can read past.',+ subscription: 'It is a wall that keeps the page\'s article or content from them until they subscribe, pay or '+ + 'start a trial, such as "Subscribe to continue reading".',+ registration: 'It is a wall that keeps the page\'s article or content from them until they create an account or '+ + 'register.',+ login: 'It is a wall that keeps the page\'s article or content from them until they sign in.',+ other: 'Something else, such as a cookie or privacy notice, an offer to subscribe or a newsletter beside content '+ + 'they can still read, an advertisement, a form to fill in, a confirmation, or a message they can read past.',}),// Each answer but `other` stops actions or limits reading, so it needs a// sure reading; a title nobody could read stops nothing.diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex b939cf390..3b0977a54 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -38,24 +38,21 @@return rect.width >= 20 && rect.height >= 20;}- function pageGateType(text) {- const value = String(text || '').toLowerCase();- if (/\bsubscribe\b|\bsubscription\b|subscriber[- ]only|unlimited access|unlock (?:this|the) article|start (?:your )?(?:free )?trial/.test(value)) return 'subscription';- if (/create (?:a )?(?:free )?account|register to (?:continue|read)|sign up to (?:continue|read)/.test(value)) return 'registration';- if (/(?:log|sign) in to (?:continue|read)|already have an account|create (?:a )?(?:free )?account or log in/.test(value)) return 'login';- return 'unknown';- }+ // What an overlay or a block in an article asks of its reader, read by a+ // model in any language from its text (the agent's page_blocker readings):+ // a subscription, registration or sign-in wall in front of the article. A+ // text nobody has read yet is no wall; it is reported for the agent to+ // read, and the agent asks again with the reading.+ const PAGE_GATE_TYPES = new Set(['subscription', 'registration', 'login']);- function pageGateHasAccessLanguage(text) {- const value = String(text || '').replace(/\s+/g, ' ').trim().toLowerCase();- if (!value) return false;- return /(?:(?:subscribe|subscription).{0,48}(?:continue|read|access|article|options|required|unlock)|subscriber[- ]only|unlimited access|unlock (?:this|the) article|start (?:your )?(?:free )?trial|create (?:a )?(?:free )?account|register to (?:continue|read)|sign up to (?:continue|read)|(?:log|sign) in to (?:continue|read)|continue reading (?:with|by)|to continue reading|already have an account)/.test(value);+ function pageGateText(el) {+ return _pageReadingText(el?.innerText || '').slice(0, 400);}- function pageGateHasInlineBlockingLanguage(text) {- const value = String(text || '').replace(/\s+/g, ' ').trim().toLowerCase();- if (!value) return false;- return /(?:to continue reading|continue reading (?:with|by)|subscriber[- ]only|(?:subscribe|subscription required|register|sign up|log in|sign in|create (?:a )?(?:free )?account).{0,64}(?:continue reading|read (?:this |the )?(?:full )?(?:article|story)|unlock (?:this|the) (?:article|story)|access (?:this|the) (?:article|story)))/.test(value);+ function pageGateReading(text) {+ if (!text) return null;+ const reading = _pageReading('page_blocker', text, null);+ return PAGE_GATE_TYPES.has(reading) ? reading : null;}function pageHasArticleContext() {@@ -71,35 +68,14 @@}}- function boundedPageGateLabel(el, rawLabel) {- const options = [];- let boundaryElement = null;- for (const value of [el.getAttribute('aria-label'), el.getAttribute('title')]) {- const normalized = String(value || '').replace(/\s+/g, ' ').trim();- if (pageGateHasAccessLanguage(normalized)) options.push(normalized);- }- let descendants = [];- try { descendants = el.querySelectorAll('h1, h2, h3, p, button, a, label, [role="heading"]'); } catch {}- for (const node of Array.from(descendants).slice(0, 80)) {- if (!gateElementIsRendered(node)) continue;- const normalized = String(node.innerText || '').replace(/\s+/g, ' ').trim();- if (normalized.length <= 600 && pageGateHasAccessLanguage(normalized)) {- boundaryElement ||= node;- options.push(normalized);- }- }- options.sort((a, b) => a.length - b.length);- if (options[0]) return { label: options[0].slice(0, 240), boundaryElement };- const accessStart = rawLabel.search(/\b(?:subscribe|subscription|subscriber|unlock|register|sign up|log in|sign in|create an? account|continue reading)\b/i);- return {- label: rawLabel.slice(Math.max(0, accessStart), Math.max(0, accessStart) + 240),- boundaryElement,- };- }-function pageGatePublic(gate) {if (!gate) return null;- return { type: gate.type, blocking: true, surface: gate.surface, label: gate.label };+ return {+ type: gate.type,+ blocking: true,+ surface: gate.surface,+ label: gate.label,+ };}function detectPageGate() {@@ -112,9 +88,9 @@for (const el of matches) {if (seen.has(el) || !gateElementIsRendered(el)) continue;seen.add(el);- const rawLabel = String(el.innerText || '').replace(/\s+/g, ' ').trim();const role = String(el.getAttribute('role') || '').toLowerCase();- let surface = (role === 'dialog' || role === 'alertdialog' || el.tagName === 'DIALOG' || el.getAttribute('aria-modal') === 'true') ? 'dialog' : 'inline';+ let surface = (role === 'dialog' || role === 'alertdialog' || el.tagName === 'DIALOG'+ || el.getAttribute('aria-modal') === 'true') ? 'dialog' : 'inline';const inArticle = !!el.closest('article, [role="article"], main, [role="main"]');let coveringOverlay = false;try {@@ -123,25 +99,24 @@const viewportArea = Math.max(1, window.innerWidth * window.innerHeight);const visibleWidth = Math.max(0, Math.min(rect.right, window.innerWidth) - Math.max(rect.left, 0));const visibleHeight = Math.max(0, Math.min(rect.bottom, window.innerHeight) - Math.max(rect.top, 0));- coveringOverlay = ['fixed', 'sticky', 'absolute'].includes(style.position) && ((visibleWidth * visibleHeight) / viewportArea) >= 0.2;- } catch {}+ coveringOverlay = ['fixed', 'sticky', 'absolute'].includes(style.position) &&+ ((visibleWidth * visibleHeight) / viewportArea) >= 0.2;+ } catch { /* rendered check already supplied safe defaults */ }if (coveringOverlay) surface = 'dialog';- if (surface === 'dialog') {- if (!articleContext || !pageGateHasAccessLanguage(rawLabel)) continue;- } else {- if (!inArticle || !pageGateHasInlineBlockingLanguage(rawLabel)) continue;- }- const gateText = boundedPageGateLabel(el, rawLabel);- const label = gateText.label;- const namedGate = /paywall|gateway|regiwall|subscription|registration/i.test([el.id, typeof el.className === 'string' ? el.className : '', el.getAttribute('data-testid') || ''].join(' '));- const score = (surface === 'dialog' ? 100 : 0) + (inArticle ? 40 : 0) + (coveringOverlay ? 30 : 0) + (namedGate ? 15 : 0) - Math.min(label.length, 2000) / 2000;- candidates.push({- element: surface === 'inline' ? (gateText.boundaryElement || el) : el,- type: pageGateType(rawLabel),- surface,- label: label.slice(0, 240),- score,- });+ // An overlay counts only on an article page, and a block only inside+ // the article; its text then has to read as a wall.+ if (surface === 'dialog' ? !articleContext : !inArticle) continue;+ const label = pageGateText(el);+ const type = pageGateReading(label);+ if (!type) continue;+ const namedGate = /paywall|gateway|regiwall|subscription|registration/i.test([+ el.id,+ typeof el.className === 'string' ? el.className : '',+ el.getAttribute('data-testid') || '',+ ].join(' '));+ const score = (surface === 'dialog' ? 100 : 0) + (inArticle ? 40 : 0) ++ (coveringOverlay ? 30 : 0) + (namedGate ? 15 : 0) - Math.min(label.length, 2000) / 2000;+ candidates.push({ element: el, type, surface, label: label.slice(0, 240), score });}}candidates.sort((a, b) => b.score - a.score);@@ -5757,6 +5732,7 @@message_field: new Set(['to', 'cc', 'bcc', 'subject', 'body', 'other']),message_author: new Set(['self', 'other']),message_status: new Set(['delivered', 'sent', 'pending', 'unclear']),+ page_blocker: new Set(['human_check', 'subscription', 'registration', 'login', 'other']),});// Readings that name an action of the control itself, never a contact's name.const _MESSAGE_CONTROL_ACTIONS = new Set(['send', 'publish', 'reply', 'new_message', 'new_post',@@ -5814,9 +5790,15 @@return reading === 'send' || reading === null;}- function _takeUnreadPageTexts() {- const texts = [..._pageReadingsUnread.values()];- _pageReadingsUnread.clear();+ // The texts consulted unread, all of them or those of one question, taken+ // from the queue for the agent to read.+ function _takeUnreadPageTexts(kind = '') {+ const texts = [];+ for (const [key, entry] of [..._pageReadingsUnread.entries()]) {+ if (kind && entry.kind !== kind) continue;+ texts.push(entry);+ _pageReadingsUnread.delete(key);+ }return texts;}@@ -9286,6 +9268,15 @@if (!value.refScopeUrl) value.refScopeUrl = location.href;} catch { /* identity is best-effort; never break the response */ }}+ // Overlay and article-block texts the page-gate check consulted unread+ // go back to the agent to be read; it asks again with the readings.+ if (['get_page_info', 'get_page_info_cdp', 'get_accessibility_tree'].includes(msg.action)+ && value && typeof value === 'object') {+ const unread = _takeUnreadPageTexts('page_blocker');+ if (unread.length) {+ value.pageTextsNeeded = [...(Array.isArray(value.pageTextsNeeded) ? value.pageTextsNeeded : []), ...unread];+ }+ }return value;};diff --git a/test/fixtures/run.mjs b/test/fixtures/run.mjsindex e6766e700..dc7161a28 100644--- a/test/fixtures/run.mjs+++ b/test/fixtures/run.mjs@@ -27,6 +27,7 @@ import {fitReadPageWindowResult as fitFirefoxReadPageWindowResult,} from '../../src/firefox/src/agent/read-page-window.js';import { CDPClient, cdpClient } from '../../src/chrome/src/cdp/cdp-client.js';+import { fixturePageBlockerReading } from '../semantic-decision-stub.mjs';import {SELECTION_SHORTCUT_LOCALES,getSelectionShortcutLocalization,@@ -303,7 +304,32 @@ async function readThroughCdpMirror(page, opts = {}) {client.evaluate = async (_tabId, expression) => ({result: { value: await page.evaluate(expression) },});- return client.readPage(1, opts);+ // The agent reads the overlay and article-block texts the mirror consulted+ // unread (a model; the fixture oracle stands in), then reads again.+ const readings = { ...(opts.pageBlockerReadings || {}) };+ let result = await client.readPage(1, { ...opts, pageBlockerReadings: readings });+ const needed = (result?.pageTextsNeeded || []).filter(entry => entry?.kind === 'page_blocker');+ if (needed.length) {+ for (const { text } of needed) readings[text] = fixturePageBlockerReading(text);+ result = await client.readPage(1, { ...opts, pageBlockerReadings: readings });+ }+ if (result && typeof result === 'object') delete result.pageTextsNeeded;+ return result;+}++// A page read the way the agent reads it: overlay and article-block texts the+// page-gate check consulted unread are read by a model (the fixture oracle+// stands in) and the page is read again with those readings.+async function readWithGateReadings(page, action, params = {}) {+ const readings = {};+ let result = await call(page, action, { ...params, __wbPageReadings: { page_blocker: readings } });+ const needed = (result?.pageTextsNeeded || []).filter(entry => entry?.kind === 'page_blocker');+ if (needed.length) {+ for (const { text } of needed) readings[text] = fixturePageBlockerReading(text);+ result = await call(page, action, { ...params, __wbPageReadings: { page_blocker: readings } });+ }+ if (result && typeof result === 'object') delete result.pageTextsNeeded;+ return result;}async function clickedSentinel(page) {@@ -1006,7 +1032,7 @@ for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']]) {test(`${label}: blocking NYTimes registration dialog suppresses article DOM`, async (page) => {await setupContentFixture(page, 'nyt-registration-gate.html', browserKind);- const result = await call(page, 'get_page_info_cdp', {});+ const result = await readWithGateReadings(page, 'get_page_info_cdp', {});if (result.pageGate?.blocking !== true || result.pageGate?.surface !== 'dialog' || result.pageGate?.type !== 'registration') {throw new Error(`registration pageGate mismatch: ${JSON.stringify(result.pageGate)}`);}@@ -1020,7 +1046,7 @@ for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']if (JSON.stringify(result).indexOf('"pageGate"') > JSON.stringify(result).indexOf('"text"')) {throw new Error('pageGate must serialize before long article text for trace visibility');}- const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxDepth: 5 });+ const tree = await readWithGateReadings(page, 'get_accessibility_tree', { filter: 'visible', maxDepth: 5 });if (tree.pageGate?.blocking !== true || !/Create a free account/i.test(tree.pageGate.label || '')) {throw new Error(`accessibility tree omitted structured pageGate: ${JSON.stringify(tree.pageGate)}`);}@@ -1037,7 +1063,7 @@ for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']if (clickResult?.success !== true || !gateControlClicked) {throw new Error(`gate control ref was not actionable: ${JSON.stringify(clickResult)}`);}- const basicResult = await call(page, 'get_page_info', {});+ const basicResult = await readWithGateReadings(page, 'get_page_info', {});if (/SECRET_NYT_(?:ARTICLE|LINK|IMAGE|FORM|SHADOW)/.test(JSON.stringify(basicResult))) {throw new Error(`basic page info leaked blocked article data: ${JSON.stringify(basicResult)}`);}@@ -1045,7 +1071,7 @@ for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']test(`${label}: The Athletic covering subscription overlay suppresses server-rendered body`, async (page) => {await setupContentFixture(page, 'athletic-subscription-overlay.html', browserKind);- const result = await call(page, 'get_page_info_cdp', {});+ const result = await readWithGateReadings(page, 'get_page_info_cdp', {});if (result.pageGate?.type !== 'subscription' || result.pageGate?.surface !== 'dialog') {throw new Error(`Athletic pageGate mismatch: ${JSON.stringify(result.pageGate)}`);}@@ -1056,7 +1082,7 @@ for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']test(`${label}: inline article gate returns only the visible preview`, async (page) => {await setupContentFixture(page, 'inline-article-paywall.html', browserKind);- const result = await call(page, 'get_page_info_cdp', {});+ const result = await readWithGateReadings(page, 'get_page_info_cdp', {});if (result.pageGate?.blocking !== true || result.pageGate?.surface !== 'inline') {throw new Error(`inline pageGate mismatch: ${JSON.stringify(result.pageGate)}`);}@@ -1066,7 +1092,7 @@ for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']if (!/\(pre-gate\)$/.test(result.textSource || '')) {throw new Error(`inline textSource missing pre-gate marker: ${result.textSource}`);}- const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxDepth: 5 });+ const tree = await readWithGateReadings(page, 'get_accessibility_tree', { filter: 'visible', maxDepth: 5 });if (!/VISIBLE_PREVIEW_PARAGRAPH/.test(tree.pageContent || '') || /SECRET_POST_GATE_PARAGRAPH/.test(tree.pageContent || '')) {throw new Error(`inline accessibility boundary mismatch: ${JSON.stringify(tree)}`);}@@ -1074,25 +1100,48 @@ for (const [label, browserKind] of [['Chrome', 'chrome'], ['Firefox', 'firefox']test(`${label}: readable article ignores header controls, inline upsells, and hidden gate markup`, async (page) => {await setupContentFixture(page, 'readable-article-no-gate.html', browserKind);- const result = await call(page, 'get_page_info_cdp', {});+ const result = await readWithGateReadings(page, 'get_page_info_cdp', {});if (result.pageGate) throw new Error(`false-positive pageGate: ${JSON.stringify(result.pageGate)}`);if (!/READABLE_ARTICLE_BODY/.test(result.text || '') || !/READABLE_ARTICLE_AFTER_UPSELL/.test(result.text || '') || result.textSource === 'page-gate') {throw new Error(`readable article body missing: ${JSON.stringify({ textSource: result.textSource, text: result.text })}`);}});+ test(`${label}: a Turkish subscription wall is read by meaning before the article is withheld`, async (page) => {+ await setupContentFixture(page, 'turkish-subscription-overlay.html', browserKind);+ // Unread, the wall decides nothing: the page reads as before and the+ // wall's text goes back to the agent to be read.+ const unread = await call(page, 'get_page_info_cdp', {});+ if (unread.pageGate || !(unread.pageTextsNeeded || []).some(entry => entry.kind === 'page_blocker'+ && /Okumaya devam etmek için abone olun/.test(entry.text))) {+ throw new Error(`unread wall text was not reported: ${JSON.stringify({ pageGate: unread.pageGate, needed: unread.pageTextsNeeded })}`);+ }+ // Read as a subscription wall, it withholds the article and its links.+ const result = await readWithGateReadings(page, 'get_page_info_cdp', {});+ if (result.pageGate?.type !== 'subscription' || result.pageGate?.surface !== 'dialog' || result.textSource !== 'page-gate') {+ throw new Error(`Turkish pageGate mismatch: ${JSON.stringify(result.pageGate)}`);+ }+ if (/SECRET_TURKISH_(?:ARTICLE|LINK)/.test(JSON.stringify(result))) {+ throw new Error(`Turkish wall leaked the article: ${JSON.stringify(result)}`);+ }+ const tree = await readWithGateReadings(page, 'get_accessibility_tree', { filter: 'visible', maxDepth: 5 });+ if (tree.pageGate?.type !== 'subscription' || /SECRET_TURKISH_ARTICLE_BODY/.test(tree.pageContent || '')) {+ throw new Error(`Turkish wall accessibility read mismatch: ${JSON.stringify(tree)}`);+ }+ });+test(`${label}: non-article signup dialog preserves form controls`, async (page) => {await setupContentFixture(page, 'non-article-signup-dialog.html', browserKind);- const result = await call(page, 'get_page_info_cdp', {});+ const result = await readWithGateReadings(page, 'get_page_info_cdp', {});if (result.pageGate) throw new Error(`non-article dialog became a page gate: ${JSON.stringify(result.pageGate)}`);if (result.forms?.length !== 1 || result.forms[0]?.inputs?.[0]?.name !== 'email') {throw new Error(`signup form was stripped from page info: ${JSON.stringify(result.forms)}`);}- const basicResult = await call(page, 'get_page_info', {});+ const basicResult = await readWithGateReadings(page, 'get_page_info', {});if (basicResult.pageGate || basicResult.forms?.length !== 1) {throw new Error(`basic page info stripped the signup form: ${JSON.stringify(basicResult)}`);}- const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxDepth: 6 });+ const tree = await readWithGateReadings(page, 'get_accessibility_tree', { filter: 'visible', maxDepth: 6 });if (tree.pageGate || !/Work email/.test(tree.pageContent || '')) {throw new Error(`signup accessibility tree was stripped: ${JSON.stringify(tree)}`);}diff --git a/test/fixtures/turkish-subscription-overlay.html b/test/fixtures/turkish-subscription-overlay.htmlnew file mode 100644index 000000000..446403d3c--- /dev/null+++ b/test/fixtures/turkish-subscription-overlay.html@@ -0,0 +1,28 @@+<!doctype html>+<html lang="tr">+<head>+ <meta charset="utf-8">+ <meta property="og:type" content="article">+ <title>Türkçe abonelik duvarı fixture</title>+ <style>+ body { margin: 0; font: 18px/1.5 sans-serif; }+ article { max-width: 700px; margin: 32px auto; }+ .abone-perdesi { position: fixed; inset: 0; z-index: 20; display: grid; place-items: center; background: rgba(255,255,255,.98); }+ .abone-kutusu { width: 560px; padding: 36px; border: 1px solid #aaa; }+ </style>+</head>+<body>+ <article>+ <h1>Fixture manşeti</h1>+ <p>SECRET_TURKISH_ARTICLE_BODY abonelik perdesinin altında sunucuda çizilmiştir ve okunmamalıdır.</p>+ <a href="https://example.com/SECRET_TURKISH_LINK_TARGET">SECRET_TURKISH_LINK_TEXT</a>+ </article>+ <div class="abone-perdesi" role="dialog" aria-modal="true">+ <div class="abone-kutusu">+ <h2>Bu içerik abonelere özeldir</h2>+ <p>Okumaya devam etmek için abone olun.</p>+ <button type="button">Abone ol</button>+ </div>+ </div>+</body>+</html>diff --git a/test/run.js b/test/run.jsindex cc9e8b6c3..2a371aa46 100644--- a/test/run.js+++ b/test/run.js@@ -18419,6 +18419,37 @@ test('a human-verification dialog in any language arms the CAPTCHA gate once a m}});+test('a page read has its unread wall texts read in any language and reads the page again', async () => {+ for (const [build, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) {+ const wall = 'Bu içerik abonelere özeldir Okumaya devam etmek için abone olun. Abone ol';+ const agent = new AgentClass({});+ const asked = stubCaptchaTitleReadings(agent);+ // The read carries the readings the agent holds, and only page reads do.+ assert.deepEqual(agent._withPageBlockerReadings('click_ax', { ref_id: 'ref_1' }), { ref_id: 'ref_1' });+ let dispatches = 0;+ const reread = await agent._rereadWithPageBlockerReadings(1, 'read_page',+ { text: 'SECRET_ARTICLE', pageTextsNeeded: [{ kind: 'page_blocker', text: wall }, { kind: 'message_status', text: 'Okundu' }] },+ async () => { dispatches += 1; return { text: wall, textSource: 'page-gate', pageGate: { type: 'subscription', blocking: true, surface: 'dialog', label: wall } }; });+ assert.equal(dispatches, 1, `${build}: the page was not read again after its wall text was read`);+ assert.equal(reread.pageGate?.type, 'subscription');+ assert.equal(reread.pageTextsNeeded, undefined);+ assert.deepEqual(asked.map(item => [item.kind, item.text]), [['page_blocker', wall]],+ `${build}: only the wall text was to be read, and once`);+ assert.deepEqual(agent._withPageBlockerReadings('read_page', {}).__wbPageReadings.page_blocker, { [wall]: 'subscription' });+ // A text already read, or one nobody can read, reads nothing more and the+ // first answer stands.+ const again = await agent._rereadWithPageBlockerReadings(1, 'get_accessibility_tree',+ { pageContent: 'tree', pageTextsNeeded: [{ kind: 'page_blocker', text: wall }] }, async () => { dispatches += 1; return {}; });+ assert.equal(dispatches, 1);+ assert.deepEqual(again, { pageContent: 'tree' });+ const unread = new AgentClass({});+ stubPageTextReadings(unread, () => null);+ const first = await unread._rereadWithPageBlockerReadings(1, 'read_page',+ { text: 'SECRET_ARTICLE', pageTextsNeeded: [{ kind: 'page_blocker', text: wall }] }, async () => { throw new Error('read again'); });+ assert.deepEqual(first, { text: 'SECRET_ARTICLE' }, `${build}: an unread wall changed the read`);+ }+});+test('Cloudflare managed-challenge signals arm only from top-level response evidence', () => {for (const [build, detector] of [['chrome', CloudflareChallengeCh],diff --git a/test/semantic-decision-stub.mjs b/test/semantic-decision-stub.mjsindex a129aeecf..0f75666be 100644--- a/test/semantic-decision-stub.mjs+++ b/test/semantic-decision-stub.mjs@@ -384,9 +384,13 @@ export function fixturePageBlockerReading(text) {const value = String(text || '').replace(/\s+/g, ' ').trim().toLowerCase();if (!value) return null;if (/captcha|security verification|human verification|verif(?:y|ication)[^.]{0,24}(?:human|robot)|are you (?:a )?human|confirm you are human|robot check|challenge verification|güvenlik doğrulaması|robot olmadığınızı|insan olduğunuzu|weryfikacja bezpieczeństwa|nie jesteś robotem|安全验证|人机验证/.test(value)) return 'human_check';- if (/subscribe|subscription|subscriber[- ]only|unlimited access|start (?:your )?(?:free )?trial|abone ol|abonelik|abonnieren|abonnement|s'abonner|abonnez-vous|suscríbete|zasubskrybuj|prenumerat/.test(value)) return 'subscription';- if (/create (?:a )?(?:free )?account|register to (?:continue|read)|sign up to (?:continue|read)|hesap oluştur|kayıt ol|konto erstellen|registrieren|créez un compte|załóż konto/.test(value)) return 'registration';- if (/(?:log|sign) in to (?:continue|read)|giriş yap(?:ın)? .*(?:okumak|devam)|anmelden,? um|connectez-vous pour|zaloguj się/.test(value)) return 'login';+ // A wall says the article stops until the reader acts; an offer beside+ // readable text does not.+ const wall = /continue reading|to read (?:this|the) (?:full )?(?:article|story)|subscriber[- ]only|unlock (?:this|the) (?:article|story)|okumaya devam|devamını okumak|weiterlesen|weiterzulesen|lire la suite|la suite est réservée|continuar leyendo|czytać dalej/.test(value);+ if (!wall) return 'other';+ if (/subscri|abone|abonn|suscr|prenumer|trial/.test(value)) return 'subscription';+ if (/create (?:a )?(?:free )?account|register|sign up|hesap oluştur|kayıt ol|konto erstellen|registrier|créez un compte|załóż konto/.test(value)) return 'registration';+ if (/log ?in|sign in|giriş yap|anmelden|connectez|zaloguj/.test(value)) return 'login';return 'other';}