Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix scroll recovery on pages with background updates

Barack Sokulluf9ec26834bab
diff --git a/package.json b/package.json
index c479b322a..012310ef1 100644
--- a/package.json
+++ b/package.json
@@ -61,8 +61,8 @@
"benchmark:relevance": "node scripts/benchmark-offline-relevance.mjs --verbose",
"test:social-contract": "node --test test/social-publish-contract.mjs test/publication-quote-markup.mjs",
"test:social-contract:dom": "node test/social-publish-contract-dom.mjs",
- "test:runtime-lifecycle": "node test/bot-memory-dom.mjs && node --test test/agent-lifecycle.mjs test/runtime-startup.mjs test/run-ui-scheduled-lifecycle.mjs test/transport-deadlines.mjs test/cdp-worker-restart.mjs test/bot-memory.mjs test/bot-memory-registries.mjs test/bot-memory-cloud-controller.mjs test/cdp-clear-receipt.mjs test/cdp-field-submit-recovery.mjs test/conversation-persistence.mjs test/history-recovery-state.mjs test/history-recovery-scheduler.mjs test/history-recovery-atomic.mjs test/context-live-tool-group.mjs test/integration-races.mjs test/watch-completion-checkpoint.mjs test/runtime-model-identity.mjs test/provider-message-privacy.mjs test/runtime-task-authority.mjs test/planner-repair-authority.mjs test/runtime-progress-continuation.mjs test/chat-resume-chain.mjs test/chat-resume-completion.mjs test/native-chat-execution-ledger.mjs test/chat-send-dispatch-result.mjs test/chat-reply-dedup-race.mjs test/memcode-lifecycle.mjs test/runtime-peer-review.mjs test/read-continuation-schema.mjs test/completion-submit-acknowledgement.mjs test/completion-live-status.mjs test/completion-iframe-submit.mjs test/form-validation-alert-readings.mjs test/control-effect-readings.mjs test/trusted-retry-readings.mjs test/completion-semantic-commit.mjs test/completion-diagnostics.mjs test/mastodon-publication.mjs test/calculator.mjs test/chat-recovery-protocol.mjs test/chat-pending-reconciliation.mjs test/semantic-page-retry.mjs test/pdf-detection.mjs test/runtime-lifecycle.mjs test/collection-read-progress.mjs test/network-lifecycle.mjs test/text-write-cancellation.mjs test/trusted-field-preparation.mjs test/draftjs-text.mjs test/rich-text-frameworks.mjs test/cdp-managed-text-selection.mjs test/firefox-rich-text-readback.mjs test/contenteditable-beforeinput.mjs test/trusted-field-preparation-ownership.mjs test/typing-preparation-geometry.mjs test/chat-steering.mjs test/lifecycle-steering-restart.mjs test/resource-mutation-guards.mjs test/completion-steering-receipt-authority.mjs test/steering-revision-recovery.mjs test/pending-clarification-hydration.mjs test/permission-steering-wakeup.mjs test/pure-calculation-reply.mjs test/pure-calculation-tools.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/ask-response-scope.mjs test/ask-completion-churn.mjs test/page-feedback-policy.mjs test/research-escalation-page-feedback.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/telegram-read-preparation-agent.mjs test/media-download-agent-handler.mjs test/code-scanning-url-recipient.mjs test/apocalypse-comm-parsing.mjs test/apocalypse-scalar-decoding.mjs test/bitgpu-chat-security.mjs test/social-media-url-upgrades.mjs test/downloaded-file-saved-bytes.mjs test/downloaded-file-local-probe.mjs test/formless-verification-recovery.mjs test/file-upload-accept.mjs test/file-upload-accept-agent.mjs test/workflow-route-generic-completion.mjs test/release-publication-completion.mjs test/workflow-implicit-metadata.mjs test/workflow-field-readings.mjs test/workflow-field-anchors.mjs test/user-turn-kind.mjs test/follow-up-task-context.mjs test/completion-current-execution-evidence.mjs test/monitor-intervention-contract.mjs test/monitor-owned-geometry.mjs test/monitor-viewport-remainder.mjs test/monitor-local-attribution.mjs test/monitor-lifecycle-leaks.mjs test/feedback-automatic-churn.mjs test/feedback-navigation-idle.mjs test/feedback-run-boundaries.mjs test/feedback-ask-viewing.mjs test/lifecycle-terminal-done.mjs test/incomplete-completion-recovery.mjs test/lifecycle-planner-digest.mjs test/lifecycle-vision-fallback.mjs test/guards-control-classification.mjs test/guards-unbound-deletion.mjs test/guards-steering-receipts.mjs && node test/ui-content-lifecycle.mjs && node --test test/native-telegram-chat.mjs test/native-telegram-recipient.mjs test/telegram-native-baseline.mjs test/telegram-native-latest.mjs test/native-chat-pre-enter-draft.mjs test/chat-draft-dispatch-ownership.mjs test/native-telegram-alias-preparation.mjs test/telegram-read-preparation-content.mjs test/queued-composer-ownership.mjs test/onboarding-provider-ready.mjs test/workflow-native-targets.mjs test/native-whatsapp-chat.mjs test/native-chat-context-identity.mjs test/native-whatsapp-contact.mjs test/whatsapp-native-bootstrap.mjs test/whatsapp-drawer-identity.mjs test/whatsapp-read-latest.mjs test/whatsapp-read-preparation.mjs test/whatsapp-chat-completion.mjs test/whatsapp-chat-restart.mjs test/chat-goal-lifecycle.mjs test/chat-completion.mjs test/chat-generic-dispatch.mjs test/completion-live-status-browser.mjs test/native-confirmation-visibility.mjs test/read-continuation-browser.mjs test/iframe-preparation-focus.mjs test/contenteditable-literal.mjs test/native-chat-empty-editor.mjs test/native-chat-trusted-clear.mjs test/chat-steering-ui.mjs test/recipient-confirmation-ui.mjs test/assistant-message-visibility-ui.mjs test/history-recovery.mjs test/history-recovery-ui.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-dialog-native.mjs test/page-monitor-inline-post.mjs test/bluesky-composer-context.mjs test/page-monitor-anchored.mjs test/page-monitor-upload-owner.mjs test/page-monitor-upload-release.mjs test/page-monitor-native-controls.mjs test/page-monitor-native-preparation.mjs test/native-preparation-cold.mjs test/page-monitor-navigation.mjs test/page-monitor-main-owner.mjs test/page-monitor-collateral-controls.mjs test/accessibility-tree-scoped-budget.mjs test/ax-reference-origin.mjs test/page-monitor-decorative-hit.mjs test/social-ui-navigation.mjs test/native-resource-deletion-binding.mjs test/native-mastodon-confirmation.mjs test/dropdown-menu-inventory.mjs test/page-monitor-passive-progress.mjs test/page-monitor-feed-items.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs test/social-media-downloader-reinjection.mjs test/upload-file-local-state-guidance.mjs && node test/message-recipient-upload.mjs && node test/message-recipient-clarify-binding.mjs && node --test test/message-recipient-whatsapp.mjs test/message-control-readings.mjs && node test/quality/integration-mv3.mjs --full-form && node test/quality/literal-calculate-midrun-mv3.mjs && node test/quality/ask-completion-churn-mv3.mjs && node test/quality/ask-person-activity-mv3.mjs && node test/quality/telegram-cold-preparation-mv3.mjs && node test/quality/native-chat-mv3.mjs && node test/quality/native-chat-presubmit-mv3.mjs && node test/quality/cloud-bot-memory-mv3.mjs && node test/quality/whatsapp-contact-matrix.mjs && node test/quality/pdf-detection-mv3.mjs && node test/quality/download-upload-mv3.mjs && node test/quality/upload-release-binding-mv3.mjs && node test/quality/release-review-world-mv3.mjs && node test/quality/downloaded-file-saved-bytes-mv3.mjs && node test/quality/file-upload-accept-mv3.mjs && node test/quality/bulk-resume-mv3.mjs && node test/quality/literal-frame-mv3.mjs && node test/quality/mastodon-publication-mv3.mjs && node test/quality/mastodon-deletion-mv3.mjs && node test/quality/mastodon-delete-confirmation-mv3.mjs && node test/quality/mastodon-delete-confirmation-mv3.mjs --menu-read && node test/quality/mastodon-delete-confirmation-mv3.mjs --vague && node test/quality/unbound-deletion-mv3.mjs && node test/quality/recipient-clarify-mv3.mjs && node test/quality/open-chat-recipients-mv3.mjs && node test/quality/correction-cross-site-mv3.mjs && node test/quality/anchored-action-mv3.mjs && node test/quality/inline-social-composer-mv3.mjs && node test/quality/bluesky-composer-mv3.mjs && node test/quality/dropdown-menu-matrix.mjs && node test/quality/mastodon-deletion-navigation-mv3.mjs && node test/quality/semantic-navigation-matrix.mjs && node test/quality/workflow-replay-mv3.mjs && node test/quality/generative-media-mv3.mjs && node test/quality/cdp-worker-restart-mv3.mjs && node test/quality/steering-worker-restart-mv3.mjs",
- "test:page-feedback": "node --test test/trusted-field-preparation.mjs test/draftjs-text.mjs test/rich-text-frameworks.mjs test/cdp-managed-text-selection.mjs test/firefox-rich-text-readback.mjs test/contenteditable-beforeinput.mjs test/trusted-field-preparation-ownership.mjs test/typing-preparation-geometry.mjs test/pure-calculation-reply.mjs test/pure-calculation-tools.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/ask-response-scope.mjs test/ask-completion-churn.mjs test/page-feedback-policy.mjs test/research-escalation-page-feedback.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/telegram-read-preparation-agent.mjs test/media-download-agent-handler.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-dialog-native.mjs test/page-monitor-inline-post.mjs test/bluesky-composer-context.mjs test/page-monitor-anchored.mjs test/page-monitor-upload-owner.mjs test/page-monitor-upload-release.mjs test/page-monitor-native-controls.mjs test/page-monitor-passive-progress.mjs test/page-monitor-feed-items.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs test/monitor-intervention-contract.mjs test/monitor-owned-geometry.mjs test/monitor-viewport-remainder.mjs test/monitor-local-attribution.mjs test/monitor-lifecycle-leaks.mjs test/feedback-automatic-churn.mjs test/feedback-navigation-idle.mjs test/feedback-run-boundaries.mjs test/feedback-ask-viewing.mjs test/delivery-click-ax-response-loss.mjs test/delivery-stop-compound.mjs test/delivery-cdp-input.mjs test/delivery-click-locks.mjs test/delivery-native-button-geometry.mjs test/delivery-hover-transition.mjs test/delivery-dispatch-boundary.mjs && node test/message-recipient-upload.mjs && node test/message-recipient-clarify-binding.mjs && node --test test/message-recipient-whatsapp.mjs",
+ "test:runtime-lifecycle": "node test/bot-memory-dom.mjs && node --test test/agent-lifecycle.mjs test/runtime-startup.mjs test/run-ui-scheduled-lifecycle.mjs test/transport-deadlines.mjs test/cdp-worker-restart.mjs test/bot-memory.mjs test/bot-memory-registries.mjs test/bot-memory-cloud-controller.mjs test/cdp-clear-receipt.mjs test/cdp-field-submit-recovery.mjs test/conversation-persistence.mjs test/history-recovery-state.mjs test/history-recovery-scheduler.mjs test/history-recovery-atomic.mjs test/context-live-tool-group.mjs test/integration-races.mjs test/watch-completion-checkpoint.mjs test/runtime-model-identity.mjs test/provider-message-privacy.mjs test/runtime-task-authority.mjs test/planner-repair-authority.mjs test/runtime-progress-continuation.mjs test/chat-resume-chain.mjs test/chat-resume-completion.mjs test/native-chat-execution-ledger.mjs test/chat-send-dispatch-result.mjs test/chat-reply-dedup-race.mjs test/memcode-lifecycle.mjs test/runtime-peer-review.mjs test/read-continuation-schema.mjs test/completion-submit-acknowledgement.mjs test/completion-live-status.mjs test/completion-iframe-submit.mjs test/form-validation-alert-readings.mjs test/control-effect-readings.mjs test/trusted-retry-readings.mjs test/completion-semantic-commit.mjs test/completion-diagnostics.mjs test/mastodon-publication.mjs test/calculator.mjs test/chat-recovery-protocol.mjs test/chat-pending-reconciliation.mjs test/semantic-page-retry.mjs test/pdf-detection.mjs test/runtime-lifecycle.mjs test/collection-read-progress.mjs test/network-lifecycle.mjs test/text-write-cancellation.mjs test/trusted-field-preparation.mjs test/draftjs-text.mjs test/rich-text-frameworks.mjs test/cdp-managed-text-selection.mjs test/firefox-rich-text-readback.mjs test/contenteditable-beforeinput.mjs test/trusted-field-preparation-ownership.mjs test/typing-preparation-geometry.mjs test/chat-steering.mjs test/lifecycle-steering-restart.mjs test/resource-mutation-guards.mjs test/completion-steering-receipt-authority.mjs test/steering-revision-recovery.mjs test/pending-clarification-hydration.mjs test/permission-steering-wakeup.mjs test/pure-calculation-reply.mjs test/pure-calculation-tools.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/ask-response-scope.mjs test/ask-completion-churn.mjs test/page-feedback-policy.mjs test/research-escalation-page-feedback.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/telegram-read-preparation-agent.mjs test/media-download-agent-handler.mjs test/code-scanning-url-recipient.mjs test/apocalypse-comm-parsing.mjs test/apocalypse-scalar-decoding.mjs test/bitgpu-chat-security.mjs test/social-media-url-upgrades.mjs test/downloaded-file-saved-bytes.mjs test/downloaded-file-local-probe.mjs test/formless-verification-recovery.mjs test/file-upload-accept.mjs test/file-upload-accept-agent.mjs test/workflow-route-generic-completion.mjs test/release-publication-completion.mjs test/workflow-implicit-metadata.mjs test/workflow-field-readings.mjs test/workflow-field-anchors.mjs test/user-turn-kind.mjs test/follow-up-task-context.mjs test/completion-current-execution-evidence.mjs test/monitor-intervention-contract.mjs test/monitor-owned-geometry.mjs test/monitor-viewport-remainder.mjs test/monitor-scroll-churn.mjs test/monitor-local-attribution.mjs test/monitor-lifecycle-leaks.mjs test/feedback-automatic-churn.mjs test/feedback-navigation-idle.mjs test/feedback-run-boundaries.mjs test/feedback-ask-viewing.mjs test/lifecycle-terminal-done.mjs test/incomplete-completion-recovery.mjs test/lifecycle-planner-digest.mjs test/lifecycle-vision-fallback.mjs test/guards-control-classification.mjs test/guards-unbound-deletion.mjs test/guards-steering-receipts.mjs && node test/ui-content-lifecycle.mjs && node --test test/native-telegram-chat.mjs test/native-telegram-recipient.mjs test/telegram-native-baseline.mjs test/telegram-native-latest.mjs test/native-chat-pre-enter-draft.mjs test/chat-draft-dispatch-ownership.mjs test/native-telegram-alias-preparation.mjs test/telegram-read-preparation-content.mjs test/queued-composer-ownership.mjs test/onboarding-provider-ready.mjs test/workflow-native-targets.mjs test/native-whatsapp-chat.mjs test/native-chat-context-identity.mjs test/native-whatsapp-contact.mjs test/whatsapp-native-bootstrap.mjs test/whatsapp-drawer-identity.mjs test/whatsapp-read-latest.mjs test/whatsapp-read-preparation.mjs test/whatsapp-chat-completion.mjs test/whatsapp-chat-restart.mjs test/chat-goal-lifecycle.mjs test/chat-completion.mjs test/chat-generic-dispatch.mjs test/completion-live-status-browser.mjs test/native-confirmation-visibility.mjs test/read-continuation-browser.mjs test/iframe-preparation-focus.mjs test/contenteditable-literal.mjs test/native-chat-empty-editor.mjs test/native-chat-trusted-clear.mjs test/chat-steering-ui.mjs test/recipient-confirmation-ui.mjs test/assistant-message-visibility-ui.mjs test/history-recovery.mjs test/history-recovery-ui.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-dialog-native.mjs test/page-monitor-inline-post.mjs test/bluesky-composer-context.mjs test/page-monitor-anchored.mjs test/page-monitor-upload-owner.mjs test/page-monitor-upload-release.mjs test/page-monitor-native-controls.mjs test/page-monitor-native-preparation.mjs test/native-preparation-cold.mjs test/page-monitor-navigation.mjs test/page-monitor-main-owner.mjs test/page-monitor-collateral-controls.mjs test/accessibility-tree-scoped-budget.mjs test/ax-reference-origin.mjs test/page-monitor-decorative-hit.mjs test/social-ui-navigation.mjs test/native-resource-deletion-binding.mjs test/native-mastodon-confirmation.mjs test/dropdown-menu-inventory.mjs test/page-monitor-passive-progress.mjs test/page-monitor-feed-items.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs test/social-media-downloader-reinjection.mjs test/upload-file-local-state-guidance.mjs && node test/message-recipient-upload.mjs && node test/message-recipient-clarify-binding.mjs && node --test test/message-recipient-whatsapp.mjs test/message-control-readings.mjs && node test/quality/integration-mv3.mjs --full-form && node test/quality/literal-calculate-midrun-mv3.mjs && node test/quality/ask-completion-churn-mv3.mjs && node test/quality/ask-person-activity-mv3.mjs && node test/quality/telegram-cold-preparation-mv3.mjs && node test/quality/native-chat-mv3.mjs && node test/quality/native-chat-presubmit-mv3.mjs && node test/quality/cloud-bot-memory-mv3.mjs && node test/quality/whatsapp-contact-matrix.mjs && node test/quality/pdf-detection-mv3.mjs && node test/quality/download-upload-mv3.mjs && node test/quality/upload-release-binding-mv3.mjs && node test/quality/release-review-world-mv3.mjs && node test/quality/downloaded-file-saved-bytes-mv3.mjs && node test/quality/file-upload-accept-mv3.mjs && node test/quality/bulk-resume-mv3.mjs && node test/quality/literal-frame-mv3.mjs && node test/quality/mastodon-publication-mv3.mjs && node test/quality/mastodon-deletion-mv3.mjs && node test/quality/mastodon-delete-confirmation-mv3.mjs && node test/quality/mastodon-delete-confirmation-mv3.mjs --menu-read && node test/quality/mastodon-delete-confirmation-mv3.mjs --vague && node test/quality/unbound-deletion-mv3.mjs && node test/quality/recipient-clarify-mv3.mjs && node test/quality/open-chat-recipients-mv3.mjs && node test/quality/correction-cross-site-mv3.mjs && node test/quality/anchored-action-mv3.mjs && node test/quality/inline-social-composer-mv3.mjs && node test/quality/bluesky-composer-mv3.mjs && node test/quality/dropdown-menu-matrix.mjs && node test/quality/mastodon-deletion-navigation-mv3.mjs && node test/quality/semantic-navigation-matrix.mjs && node test/quality/workflow-replay-mv3.mjs && node test/quality/generative-media-mv3.mjs && node test/quality/cdp-worker-restart-mv3.mjs && node test/quality/steering-worker-restart-mv3.mjs",
+ "test:page-feedback": "node --test test/trusted-field-preparation.mjs test/draftjs-text.mjs test/rich-text-frameworks.mjs test/cdp-managed-text-selection.mjs test/firefox-rich-text-readback.mjs test/contenteditable-beforeinput.mjs test/trusted-field-preparation-ownership.mjs test/typing-preparation-geometry.mjs test/pure-calculation-reply.mjs test/pure-calculation-tools.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/ask-response-scope.mjs test/ask-completion-churn.mjs test/page-feedback-policy.mjs test/research-escalation-page-feedback.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/telegram-read-preparation-agent.mjs test/media-download-agent-handler.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-dialog-native.mjs test/page-monitor-inline-post.mjs test/bluesky-composer-context.mjs test/page-monitor-anchored.mjs test/page-monitor-upload-owner.mjs test/page-monitor-upload-release.mjs test/page-monitor-native-controls.mjs test/page-monitor-passive-progress.mjs test/page-monitor-feed-items.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs test/monitor-intervention-contract.mjs test/monitor-owned-geometry.mjs test/monitor-viewport-remainder.mjs test/monitor-scroll-churn.mjs test/monitor-local-attribution.mjs test/monitor-lifecycle-leaks.mjs test/feedback-automatic-churn.mjs test/feedback-navigation-idle.mjs test/feedback-run-boundaries.mjs test/feedback-ask-viewing.mjs test/delivery-click-ax-response-loss.mjs test/delivery-stop-compound.mjs test/delivery-cdp-input.mjs test/delivery-click-locks.mjs test/delivery-native-button-geometry.mjs test/delivery-hover-transition.mjs test/delivery-dispatch-boundary.mjs && node test/message-recipient-upload.mjs && node test/message-recipient-clarify-binding.mjs && node --test test/message-recipient-whatsapp.mjs",
"test:history-recovery": "node --test test/history-recovery.mjs test/history-recovery-ui.mjs test/history-recovery-state.mjs test/history-recovery-scheduler.mjs test/history-recovery-atomic.mjs test/conversation-persistence.mjs",
"test:chat-steering": "node --test test/chat-steering.mjs test/chat-steering-ui.mjs test/recipient-confirmation-ui.mjs test/lifecycle-steering-latency.mjs test/lifecycle-steering-restart.mjs test/lifecycle-run-ui.mjs test/lifecycle-menu-prompts-held.mjs",
"test:rich-text-dom": "node test/ui-content-lifecycle.mjs --dom",
@@ -96,7 +96,7 @@
"test:upload-owner": "node --test test/page-monitor-upload-owner.mjs test/page-monitor-upload-release.mjs",
"test:shard:quick": "npm run test:cms && npm run test:markdown && npm run test:generative-media && npm run test:systemone && npm run test:systemone:fast && node --test test/browser-dialogs.mjs test/capsolver-cloud-broker.mjs test/captcha-providers.mjs test/captcha-weighted-fallback.mjs test/captcha-hcaptcha-providers.mjs test/captcha-native-providers.mjs test/captcha-runtime-policy.mjs test/captcha-callback-bridge.mjs test/captcha-bridge-registration.mjs test/captcha-aws-recovery.mjs test/captcha-documented-contracts.mjs test/huggingface-signup-recovery.mjs test/trace-full-export.mjs test/feedback-traces.mjs && npm run test:firefox-bidi && npm run test:billing-notice && npm run test:compass-quota && npm run test:provider-limits && npm run test:provider-catalog && npm run test:provider-instances:ui && npm run test:openrouter-capabilities && npm run test:anthropic && npm run test:browser-batch && npm run test:accessibility-tree-benchmark && npm run test:toolbar-guard && npm run test:cloud-bridge-approval && npm run test:discord && npm run test:pdf-read && npm run test:pdf-selection && npm run test:social-contract && npm run test:safesocial && npm run test:build-unpacked && npm run test:attachment-drop && npm run test:workflow-editor && node --test test/profile-sync-stats.mjs && node --test test/memcode-recall.mjs",
"test:shard:runtime-unit": "node --test test/skill-download-wait.mjs && node --test test/agent-lifecycle.mjs test/runtime-startup.mjs test/run-ui-scheduled-lifecycle.mjs test/transport-deadlines.mjs test/cdp-worker-restart.mjs test/bot-memory.mjs test/bot-memory-click-guard.mjs test/bot-memory-registries.mjs test/bot-memory-cloud-controller.mjs test/cdp-clear-receipt.mjs test/cdp-field-submit-recovery.mjs test/conversation-persistence.mjs test/context-live-tool-group.mjs test/integration-races.mjs test/watch-completion-checkpoint.mjs test/runtime-model-identity.mjs test/provider-message-privacy.mjs test/runtime-task-authority.mjs test/planner-repair-authority.mjs test/runtime-progress-continuation.mjs test/chat-resume-chain.mjs test/chat-resume-completion.mjs test/native-chat-execution-ledger.mjs test/chat-send-dispatch-result.mjs test/chat-reply-dedup-race.mjs test/memcode-lifecycle.mjs test/runtime-peer-review.mjs test/read-continuation-schema.mjs test/completion-submit-acknowledgement.mjs test/completion-live-status.mjs test/completion-iframe-submit.mjs test/form-validation-alert-readings.mjs test/control-effect-readings.mjs test/trusted-retry-readings.mjs test/completion-semantic-commit.mjs test/completion-diagnostics.mjs test/mastodon-publication.mjs test/calculator.mjs test/chat-recovery-protocol.mjs test/chat-pending-reconciliation.mjs test/semantic-page-retry.mjs test/pdf-detection.mjs test/runtime-lifecycle.mjs test/collection-read-progress.mjs test/network-lifecycle.mjs test/text-write-cancellation.mjs test/trusted-field-preparation.mjs test/draftjs-text.mjs test/rich-text-frameworks.mjs test/cdp-managed-text-selection.mjs test/firefox-rich-text-readback.mjs test/contenteditable-beforeinput.mjs test/trusted-field-preparation-ownership.mjs test/typing-preparation-geometry.mjs test/chat-steering.mjs test/resource-mutation-guards.mjs test/completion-steering-receipt-authority.mjs test/steering-revision-recovery.mjs test/pending-clarification-hydration.mjs test/permission-steering-wakeup.mjs test/pure-calculation-reply.mjs test/pure-calculation-tools.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/ask-response-scope.mjs test/ask-completion-churn.mjs test/page-feedback-policy.mjs test/research-escalation-page-feedback.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/telegram-read-preparation-agent.mjs test/media-download-agent-handler.mjs test/code-scanning-url-recipient.mjs test/apocalypse-comm-parsing.mjs test/apocalypse-scalar-decoding.mjs test/bitgpu-chat-security.mjs test/social-media-url-upgrades.mjs test/downloaded-file-saved-bytes.mjs test/downloaded-file-local-probe.mjs test/formless-verification-recovery.mjs test/file-upload-accept.mjs test/file-upload-accept-agent.mjs test/workflow-route-generic-completion.mjs test/release-publication-completion.mjs test/workflow-implicit-metadata.mjs test/workflow-field-readings.mjs test/workflow-field-anchors.mjs test/user-turn-kind.mjs test/follow-up-task-context.mjs test/completion-current-execution-evidence.mjs test/video-upload-attachment.mjs test/feedback-automatic-churn.mjs test/feedback-navigation-idle.mjs test/feedback-run-boundaries.mjs test/feedback-ask-viewing.mjs test/lifecycle-terminal-done.mjs test/incomplete-completion-recovery.mjs test/lifecycle-planner-digest.mjs test/lifecycle-steering-latency.mjs test/lifecycle-steering-restart.mjs test/lifecycle-vision-fallback.mjs test/delivery-click-locks.mjs test/guards-steering-receipts.mjs test/terminal-user-messages.mjs && npm run test:plan-notifications",
- "test:shard:browser-a": "node --test test/captcha-draft-content.mjs test/captcha-challenge-surfaces.mjs && node test/bot-memory-dom.mjs && node test/bot-memory-click-dom.mjs && node test/ui-content-lifecycle.mjs && node --test test/native-telegram-chat.mjs test/native-telegram-recipient.mjs test/telegram-native-baseline.mjs test/telegram-native-latest.mjs test/native-chat-pre-enter-draft.mjs test/chat-draft-dispatch-ownership.mjs test/native-telegram-alias-preparation.mjs test/telegram-read-preparation-content.mjs test/queued-composer-ownership.mjs test/onboarding-provider-ready.mjs test/workflow-native-targets.mjs test/native-whatsapp-chat.mjs test/native-chat-context-identity.mjs test/native-whatsapp-contact.mjs test/whatsapp-native-bootstrap.mjs test/whatsapp-drawer-identity.mjs test/whatsapp-read-latest.mjs test/whatsapp-read-preparation.mjs test/whatsapp-chat-completion.mjs test/whatsapp-chat-restart.mjs test/chat-goal-lifecycle.mjs test/chat-completion.mjs test/chat-generic-dispatch.mjs test/completion-live-status-browser.mjs test/native-confirmation-visibility.mjs test/native-button-preparation.mjs test/read-continuation-browser.mjs test/iframe-preparation-focus.mjs test/contenteditable-literal.mjs test/native-chat-empty-editor.mjs test/native-chat-trusted-clear.mjs test/chat-steering-ui.mjs test/recipient-confirmation-ui.mjs test/assistant-message-visibility-ui.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-dialog-native.mjs test/page-monitor-inline-post.mjs test/bluesky-composer-context.mjs test/page-monitor-anchored.mjs test/page-monitor-upload-owner.mjs test/page-monitor-upload-release.mjs test/page-monitor-native-controls.mjs test/page-monitor-native-preparation.mjs test/native-preparation-cold.mjs test/page-monitor-navigation.mjs test/page-monitor-main-owner.mjs test/page-monitor-collateral-controls.mjs test/accessibility-tree-scoped-budget.mjs test/ax-reference-origin.mjs test/page-monitor-decorative-hit.mjs test/social-ui-navigation.mjs test/native-resource-deletion-binding.mjs test/native-mastodon-confirmation.mjs test/dropdown-menu-inventory.mjs test/page-monitor-passive-progress.mjs test/page-monitor-feed-items.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs test/social-media-downloader-reinjection.mjs test/upload-file-local-state-guidance.mjs test/monitor-intervention-contract.mjs test/monitor-owned-geometry.mjs test/monitor-viewport-remainder.mjs test/monitor-local-attribution.mjs test/monitor-lifecycle-leaks.mjs test/lifecycle-run-ui.mjs test/lifecycle-menu-prompts-held.mjs test/delivery-cdp-input.mjs test/delivery-click-ax-response-loss.mjs test/delivery-native-button-geometry.mjs test/delivery-hover-transition.mjs test/delivery-dispatch-boundary.mjs test/delivery-stop-compound.mjs test/guards-control-classification.mjs test/guards-unbound-deletion.mjs test/submit-probe-readings.mjs",
+ "test:shard:browser-a": "node --test test/captcha-draft-content.mjs test/captcha-challenge-surfaces.mjs && node test/bot-memory-dom.mjs && node test/bot-memory-click-dom.mjs && node test/ui-content-lifecycle.mjs && node --test test/native-telegram-chat.mjs test/native-telegram-recipient.mjs test/telegram-native-baseline.mjs test/telegram-native-latest.mjs test/native-chat-pre-enter-draft.mjs test/chat-draft-dispatch-ownership.mjs test/native-telegram-alias-preparation.mjs test/telegram-read-preparation-content.mjs test/queued-composer-ownership.mjs test/onboarding-provider-ready.mjs test/workflow-native-targets.mjs test/native-whatsapp-chat.mjs test/native-chat-context-identity.mjs test/native-whatsapp-contact.mjs test/whatsapp-native-bootstrap.mjs test/whatsapp-drawer-identity.mjs test/whatsapp-read-latest.mjs test/whatsapp-read-preparation.mjs test/whatsapp-chat-completion.mjs test/whatsapp-chat-restart.mjs test/chat-goal-lifecycle.mjs test/chat-completion.mjs test/chat-generic-dispatch.mjs test/completion-live-status-browser.mjs test/native-confirmation-visibility.mjs test/native-button-preparation.mjs test/read-continuation-browser.mjs test/iframe-preparation-focus.mjs test/contenteditable-literal.mjs test/native-chat-empty-editor.mjs test/native-chat-trusted-clear.mjs test/chat-steering-ui.mjs test/recipient-confirmation-ui.mjs test/assistant-message-visibility-ui.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-dialog-native.mjs test/page-monitor-inline-post.mjs test/bluesky-composer-context.mjs test/page-monitor-anchored.mjs test/page-monitor-upload-owner.mjs test/page-monitor-upload-release.mjs test/page-monitor-native-controls.mjs test/page-monitor-native-preparation.mjs test/native-preparation-cold.mjs test/page-monitor-navigation.mjs test/page-monitor-main-owner.mjs test/page-monitor-collateral-controls.mjs test/accessibility-tree-scoped-budget.mjs test/ax-reference-origin.mjs test/page-monitor-decorative-hit.mjs test/social-ui-navigation.mjs test/native-resource-deletion-binding.mjs test/native-mastodon-confirmation.mjs test/dropdown-menu-inventory.mjs test/page-monitor-passive-progress.mjs test/page-monitor-feed-items.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs test/social-media-downloader-reinjection.mjs test/upload-file-local-state-guidance.mjs test/monitor-intervention-contract.mjs test/monitor-owned-geometry.mjs test/monitor-viewport-remainder.mjs test/monitor-scroll-churn.mjs test/monitor-local-attribution.mjs test/monitor-lifecycle-leaks.mjs test/lifecycle-run-ui.mjs test/lifecycle-menu-prompts-held.mjs test/delivery-cdp-input.mjs test/delivery-click-ax-response-loss.mjs test/delivery-native-button-geometry.mjs test/delivery-hover-transition.mjs test/delivery-dispatch-boundary.mjs test/delivery-stop-compound.mjs test/guards-control-classification.mjs test/guards-unbound-deletion.mjs test/submit-probe-readings.mjs",
"test:shard:browser-b1": "node test/message-recipient-upload.mjs && node test/message-recipient-clarify-binding.mjs && node --test test/message-recipient-whatsapp.mjs test/message-control-readings.mjs && node test/quality/integration-mv3.mjs --full-form && node test/quality/integration-mv3.mjs --batch && node test/quality/literal-calculate-midrun-mv3.mjs && node test/quality/ask-completion-churn-mv3.mjs && node test/quality/ask-person-activity-mv3.mjs && node test/quality/telegram-cold-preparation-mv3.mjs && node test/quality/native-chat-mv3.mjs && node test/quality/native-chat-presubmit-mv3.mjs && node test/quality/cloud-bot-memory-mv3.mjs",
"test:shard:browser-b2": "node test/quality/whatsapp-contact-matrix.mjs && node test/quality/pdf-detection-mv3.mjs && node test/quality/download-upload-mv3.mjs && node test/quality/upload-release-binding-mv3.mjs && node test/quality/release-review-world-mv3.mjs && node test/quality/downloaded-file-saved-bytes-mv3.mjs && node test/quality/file-upload-accept-mv3.mjs && node test/quality/bulk-resume-mv3.mjs && node test/quality/shared-action-guards-mv3.mjs && node test/quality/video-upload-attachment-mv3.mjs",
"test:shard:browser-c": "node test/quality/literal-frame-mv3.mjs && node test/quality/mastodon-publication-mv3.mjs && node test/quality/mastodon-deletion-mv3.mjs && node test/quality/mastodon-delete-confirmation-mv3.mjs && node test/quality/mastodon-delete-confirmation-mv3.mjs --menu-read && node test/quality/mastodon-delete-confirmation-mv3.mjs --vague && node test/quality/unbound-deletion-mv3.mjs && node test/quality/recipient-clarify-mv3.mjs && node test/quality/open-chat-recipients-mv3.mjs && node test/quality/correction-cross-site-mv3.mjs && node test/quality/anchored-action-mv3.mjs && node test/quality/inline-social-composer-mv3.mjs && node test/quality/bluesky-composer-mv3.mjs && node test/quality/dropdown-menu-matrix.mjs && node test/quality/mastodon-deletion-navigation-mv3.mjs && node test/quality/semantic-navigation-matrix.mjs && node test/quality/workflow-replay-mv3.mjs && node test/quality/generative-media-mv3.mjs && node test/quality/cdp-worker-restart-mv3.mjs && node test/quality/steering-worker-restart-mv3.mjs",
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 5c1adb1da..82ebe5308 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -46887,9 +46887,6 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
) ? await this._currentUrl(tabId) : '';
if (name === 'click') throwIfEarlyCdpAborted();
- if (name === 'scroll') {
- args = await this._augmentScrollArgsWithLastInteraction(tabId, args);
- }
const contentActionDeadlineMs = this._contentActionDeadlineMs(name, args);
const contentActionStartedAt = Date.now();
const remainingContentActionDeadlineMs = () => Math.max(
diff --git a/src/chrome/src/agent/page-feedback-policy.js b/src/chrome/src/agent/page-feedback-policy.js
index a1d4b898d..1129be7f4 100644
--- a/src/chrome/src/agent/page-feedback-policy.js
+++ b/src/chrome/src/agent/page-feedback-policy.js
@@ -1,5 +1,6 @@
/** Tool freshness policy for passive page observations. Mirror in Firefox. */
import { calculateExpression, calculationExpressionSpans, calculationResultInText, tokenizeCalculationExpression } from './calculator.js';
+import { BATCH_TRAILING_OBSERVATION_TOOLS, MAX_BATCH_ACTIONS } from './browser-batch.js';
const FRESH_READS = new Set([
'get_accessibility_tree', 'get_interactive_elements', 'read_page', 'read_page_source',
@@ -255,6 +256,31 @@ function axTargetContext(page, refId) {
export function pageFeedbackCallPolicy(name, args = {}, state = {}, currentPage = null, events = []) {
if (FRESH_READS.has(name)) return { kind: 'observe' };
if (DETACHED_TOOLS.has(name)) return { kind: 'independent' };
+ // Scrolling explores the current document. A counter or animation changing
+ // while the model thinks does not invalidate that intent. The content monitor
+ // resolves its live pane and checks intervention, viewport and target identity
+ // again at dispatch; this does not grant any click or input action.
+ if (name === 'scroll' && (args.frameId == null || args.frameId === 0)
+ && (args.targetFrame == null || args.targetFrame === 0)
+ && !args.allFrames && !args.all && !args.urlFilter) return { kind: 'scroll' };
+ // The parent is checked before expansion. Retain only scrolling followed by
+ // ordinary observations; each child still passes the normal batch validator
+ // and its own preparation/dispatch checks. Never retain an arbitrary batch.
+ if (name === 'browser_batch' && Array.isArray(args.actions) && args.actions.length > 0
+ && args.actions.length <= MAX_BATCH_ACTIONS.full) {
+ let observed = false, scrolled = false;
+ const scrollBatch = args.actions.every(action => {
+ if (!action || typeof action !== 'object' || Array.isArray(action)
+ || Object.keys(action).some(key => !['name', 'input'].includes(key))
+ || !action.input || typeof action.input !== 'object' || Array.isArray(action.input)) return false;
+ if (BATCH_TRAILING_OBSERVATION_TOOLS.has(action.name)) { observed = true; return true; }
+ if (observed || action.name !== 'scroll'
+ || pageFeedbackCallPolicy('scroll', action.input).kind !== 'scroll') return false;
+ scrolled = true;
+ return true;
+ });
+ if (scrollBatch && scrolled) return { kind: 'scroll' };
+ }
if (['fetch_url', 'research_url'].includes(name) && httpUrl(args.url)
&& ['GET', 'HEAD'].includes(String(args.method || 'GET').toUpperCase())
&& !args.replayRequestId && !args.apiReplayRequestId && !args.body) return { kind: 'independent' };
diff --git a/src/chrome/src/agent/page-feedback-recovery.js b/src/chrome/src/agent/page-feedback-recovery.js
index 5586fa225..640c9811d 100644
--- a/src/chrome/src/agent/page-feedback-recovery.js
+++ b/src/chrome/src/agent/page-feedback-recovery.js
@@ -5,8 +5,8 @@ export const PAGE_FEEDBACK_NUDGE_AT = 2;
export const PAGE_FEEDBACK_STOP_AT = 5;
export const PAGE_FEEDBACK_STOP_CODE = 'page_unstable';
-const STOP_MESSAGE = 'Stopped because the page kept changing before the next action could be safely dispatched. '
- + 'The task is incomplete; no stale action was dispatched. Wait for the page to settle, then choose Retry.';
+const STOP_MESSAGE = 'WebBrain could not validate the next action after repeated browser updates. '
+ + 'The task is incomplete; no stale action was dispatched. Choose Retry to inspect the current page and continue.';
const NUDGE_MESSAGE = '[PAGE KEEPS CHANGING: Browser updates repeatedly prevented the next action from being dispatched. '
+ 'Use the fresh observation to choose an action whose intended target can be verified. '
+ 'Do not repeat stale targets or coordinates.]';
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index 6d6231624..fec5127b9 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -221,7 +221,8 @@ function preparationVetoDiagnostics(veto) {
if (new Set(['semantic_binding_changed', 'geometry_changed', 'model_snapshot_invalid', 'model_snapshot_expired',
'focus_changed', 'pending_feedback', 'revision_changed', 'document_changed', 'document_revision_changed',
'intervention_changed', 'target_changed', 'target_hit_changed', 'operation_expired', 'operation_missing',
- 'run_ended', 'target_unresolved', 'target_uncovered', 'current_owner_changed']).has(veto.predicate)) facts.predicate = veto.predicate;
+ 'run_ended', 'target_unresolved', 'target_uncovered', 'current_owner_changed',
+ 'operation_kind_changed', 'scroll_target_changed']).has(veto.predicate)) facts.predicate = veto.predicate;
return facts;
}
@@ -343,6 +344,10 @@ function safeAction(tool, args = {}) {
// click({text}) is a target label, never typed text or a field value.
if (['click', 'iframe_click'].includes(tool) && typeof args.text === 'string') target.textMatch = args.text;
for (const key of ['x', 'y']) if (Number.isFinite(args[key])) target[key] = args[key];
+ if (tool === 'scroll') {
+ if (typeof args.direction === 'string') target.direction = args.direction;
+ if (typeof args.alsoWindow === 'boolean') target.alsoWindow = args.alsoWindow;
+ }
return { tool, ...target };
}
@@ -1165,6 +1170,10 @@ export const pageFeedbackMethods = {
_pageFeedbackBlocksCall(tabId, name, args = {}) {
const run = this._pageFeedbackRuns?.get(tabId);
+ if (name === 'scroll' && run?.modelState
+ && (this._hasPendingSteering?.(tabId)
+ || !this._pageFeedbackStateCurrent(tabId, run, run.modelState, run.url, { rootOnly: true })
+ || (run.dispatchCall?.name === name && run.dispatchCall.modelState !== run.modelState))) return true;
if (run?.dispatchCall?.expectedModelSnapshot
&& (run.modelState !== run.dispatchCall.modelState
|| !this._pageFeedbackStateCurrent(tabId, run, run.dispatchCall.modelState))) return true;
@@ -1173,12 +1182,12 @@ export const pageFeedbackMethods = {
const policy = pageFeedbackCallPolicy(name, args, run.modelState, run.latestPage, events);
// Automatic churn (a superset of relaxed-retry DOM) revalidates per policy.
if (!isAutomaticPageFeedback(events) || !this._pageFeedbackStateCurrent(tabId, run, run.modelState, run.url,
- { rootOnly: policy.kind === 'request' })) return true;
+ { rootOnly: ['request', 'scroll'].includes(policy.kind) })) return true;
// Unknown DOM is observed while an agent action is in flight. It is that
// call's own uncertain effect only after the call sent input (typeahead
// renders between keys). Observed during an earlier action, or during this
// call's preparation, it stays a fence: re-observe before new input.
- if (!this._monitoringRetryCurrent(tabId, run) && events.some(event => (unknownAttribution.get(event) || [])
+ if (policy.kind !== 'scroll' && !this._monitoringRetryCurrent(tabId, run) && events.some(event => (unknownAttribution.get(event) || [])
.some(({ call, afterInput }) => call && call.mutation !== false && !(call === run.dispatchCall && afterInput)))) return true;
const policyEvents = this._monitoringRetryCurrent(tabId, run)
? events.map(event => event.source === 'unknown' && event.kind === 'dom' ? { ...event, source: 'page' } : event) : events;
@@ -1353,12 +1362,12 @@ export const pageFeedbackMethods = {
documentMatches = state.documentId ? documentId === state.documentId : state.frames.has(0);
}
const currentRun = this._pageFeedbackRuns.get(tabId) === run;
- const concreteRequests = !!responseToolCalls?.length && responseToolCalls.every(call => {
- try { return pageFeedbackCallPolicy(call.function?.name, JSON.parse(call.function.arguments)).kind === 'request'; }
+ const rootScopedCalls = !!responseToolCalls?.length && responseToolCalls.every(call => {
+ try { return ['request', 'scroll'].includes(pageFeedbackCallPolicy(call.function?.name, JSON.parse(call.function.arguments)).kind); }
catch { return false; }
});
const stateCurrent = documentMatches && currentRun
- ? this._pageFeedbackStateCurrent(tabId, run, state, url, { rootOnly: concreteRequests }) : null;
+ ? this._pageFeedbackStateCurrent(tabId, run, state, url, { rootOnly: rootScopedCalls }) : null;
const sameDocument = documentMatches && currentRun && stateCurrent;
// A tool-free Ask answer or a strictly owned literal arithmetic-only task
// is independent of passive churn. Task-only scope permits normal protocol
@@ -1500,10 +1509,19 @@ export function installPageFeedback(Agent) {
const execute = Agent.prototype.executeTool;
Agent.prototype.executeTool = async function(tabId, name, args, onUpdate, executionContext) {
const run = this._pageFeedbackRuns?.get(tabId);
+ // Resolve the browser's last-interaction hint before monitor preparation so
+ // the background and local scroll bind the same requested pane.
+ if (name === 'scroll' && this._augmentScrollArgsWithLastInteraction) {
+ args = await this._augmentScrollArgsWithLastInteraction(tabId, args);
+ }
const mutation = this.constructor.STATE_CHANGE_TOOLS.has(name) || ['upload_file', 'solve_captcha', 'apply_captcha_solution'].includes(name);
// A model cannot supply the private expected binding. Even without queued
// feedback, use the identity captured before inference for a focused download.
const modelState = run?.modelState;
+ const scrollStateCurrent = () => this._pageFeedbackRuns?.get(tabId) === run
+ && run.modelState === modelState && !this._hasPendingSteering?.(tabId)
+ && this._pageFeedbackStateCurrent(tabId, run, modelState, run.url, { rootOnly: true });
+ if (run && modelState && name === 'scroll' && !scrollStateCurrent()) return pageFeedbackPendingResult();
const batchId = executionContext?._browserBatch?.parentId;
const policy = pageFeedbackCallPolicy(name, args || {}, modelState, run?.latestPage);
if (run && policy.kind === 'media_resolve') return this._resolvePageFeedbackMedia(tabId, policy.target);
@@ -1521,8 +1539,8 @@ export function installPageFeedback(Agent) {
modelBound = validated === true;
}
const targetValidated = !!run && run.validatedTargets?.get(`${name}:${JSON.stringify(args)}`) === run.revision;
- const allowPassiveRebase = modelBound || targetValidated || (policy.kind === 'navigate'
- && !!run && this._pageFeedbackStateCurrent(tabId, run, run.modelState));
+ const allowPassiveRebase = modelBound || targetValidated || (['navigate', 'scroll'].includes(policy.kind)
+ && !!run && this._pageFeedbackStateCurrent(tabId, run, run.modelState, run.url, { rootOnly: policy.kind === 'scroll' }));
if (run && mutation && !modelBound && this._pageFeedbackBlocksCall(tabId, name, args)) return pageFeedbackPendingResult();
const dispatchState = executionContext?._contentActionDispatchState || { started: false };
executionContext = { ...executionContext, _contentActionDispatchState: dispatchState,
@@ -1560,10 +1578,12 @@ export function installPageFeedback(Agent) {
if ((modelBound ? prepared?.ready === true && prepared.modelBindingValid === true
&& this._pageFeedbackRuns?.get(tabId) === run && run.modelState === modelState
&& this._pageFeedbackStateCurrent(tabId, run, modelState)
- : run.revision === revision || policy.kind === 'navigate') && allowPassiveRebase) run.dispatchCall.allowPassiveRebase = true;
+ : policy.kind === 'scroll' ? scrollStateCurrent()
+ : run.revision === revision || policy.kind === 'navigate') && allowPassiveRebase) run.dispatchCall.allowPassiveRebase = true;
}
try {
if (modelBound && !run.dispatchCall.allowPassiveRebase) { noDispatch = true; return pageFeedbackPendingResult(); }
+ if (run && modelState && name === 'scroll' && !scrollStateCurrent()) { noDispatch = true; return pageFeedbackPendingResult(); }
if (run && mutation && this._pageFeedbackBlocksCall(tabId, name, args)) return pageFeedbackPendingResult();
const observationDocument = run?.frames.get(0)?.token, observationUrl = run?.url;
// Completion verification reads the page through this wrapper. Neither
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 50c581d99..ee62ae060 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -3079,7 +3079,7 @@
if (target) {
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target });
dispatched = true;
if (direction === 'down') target.scrollBy(0, amount);
else if (direction === 'up') target.scrollBy(0, -amount);
@@ -3091,7 +3091,7 @@
// Always also scroll the window in case both are needed (some pages have
// both window and container scrolling).
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: document.scrollingElement || document.documentElement });
dispatched = true;
if (direction === 'down') window.scrollBy(0, amount);
else if (direction === 'up') window.scrollBy(0, -amount);
@@ -3280,7 +3280,7 @@
if (target) {
containerBefore = target.scrollTop;
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target });
dispatched = true;
scrollElementInstant(target, direction, amount);
containerAfter = target.scrollTop;
@@ -3291,7 +3291,7 @@
const shouldScrollWindow = params.alsoWindow === true || !movedContainer;
if (shouldScrollWindow && windowCanMove) {
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: scrollingElement });
dispatched = true;
if (direction === 'down') window.scrollBy(0, amount);
else if (direction === 'up') window.scrollBy(0, -amount);
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 8e4cc2e81..941f04716 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -2260,7 +2260,106 @@
function ownedPreparationPhase(op) {
return op?.pointerPreparationOnly === true || op?.typingPreparationOnly === true || op?.selectorPreparationOnly === true;
}
+ // Scrolling is a relative view operation. Feed contents and uncertain DOM
+ // attribution are not its authority: the current document, human ownership
+ // and requested scroll pane are. Keep this proof separate from certificates
+ // for clicks and input, which must still bind the observed page semantics.
+ function scrollPaneFor(origin) {
+ const documentScrollable = Math.max(document.body?.scrollHeight || 0, document.documentElement?.scrollHeight || 0,
+ document.scrollingElement?.scrollHeight || 0) > window.innerHeight + 10;
+ const skipOrigin = /^(INPUT|TEXTAREA|SELECT)$/.test(origin?.tagName || '') || origin?.isContentEditable;
+ for (let node = skipOrigin ? bindingParent(origin) : origin; node && node !== document.body && node !== document.documentElement; node = bindingParent(node)) {
+ const style = getComputedStyle(node);
+ if (node.scrollHeight > node.clientHeight + 10 && (/^(auto|scroll|overlay)$/.test(style.overflowY)
+ || (!documentScrollable && style.overflowY === 'hidden'))) return node;
+ }
+ return document.scrollingElement || document.documentElement;
+ }
+ // Match smartScrollPage's last resort when neither the origin's ancestors
+ // nor the window can move. Capture that recipient before dispatch rather
+ // than granting arbitrary containers permission to borrow the origin.
+ function scrollFallbackPane(origin, direction) {
+ const documentHeight = Math.max(document.body?.scrollHeight || 0, document.documentElement?.scrollHeight || 0,
+ document.scrollingElement?.scrollHeight || 0);
+ const allowHidden = documentHeight <= window.innerHeight + 10;
+ const canMove = (position, maximum) => direction === 'down' || direction === 'bottom' ? position < maximum - 1
+ : direction === 'up' || direction === 'top' ? position > 1 : false;
+ const scrollable = node => {
+ if (!node || node === document.body || node === document.documentElement
+ || node.scrollHeight <= node.clientHeight + 10) return false;
+ const overflow = getComputedStyle(node).overflowY;
+ return (/^(auto|scroll|overlay)$/.test(overflow) || (allowHidden && overflow === 'hidden'))
+ && canMove(node.scrollTop, Math.max(0, node.scrollHeight - node.clientHeight));
+ };
+ const skipOrigin = /^(INPUT|TEXTAREA|SELECT)$/.test(origin?.tagName || '') || origin?.isContentEditable;
+ for (let node = skipOrigin ? origin.parentElement : origin; node && node !== document.body && node !== document.documentElement; node = node.parentElement) {
+ if (scrollable(node)) return null;
+ }
+ if (canMove(window.scrollY, Math.max(0, documentHeight - window.innerHeight))) return null;
+ let best = null, bestArea = 0;
+ for (const node of document.querySelectorAll('div, section, main, article, aside, [role="main"], [role="dialog"], [role="region"], [role="listbox"], [role="menu"]')) {
+ if (!scrollable(node)) continue;
+ const rect = node.getBoundingClientRect();
+ if (rect.width < 1 || rect.height < 1 || rect.bottom < 0 || rect.top > window.innerHeight || rect.right < 0 || rect.left > window.innerWidth) continue;
+ const area = rect.width * rect.height;
+ if (area > bestArea) { best = node; bestArea = area; }
+ }
+ return bestArea > window.innerWidth * window.innerHeight * 0.15 ? best : null;
+ }
+ function scrollPaneRectMatches(pane, before) {
+ const current = rectFor(pane);
+ return !!current && !!before && ['x', 'y', 'width', 'height'].every(key => Math.abs(current[key] - before[key]) <= 2);
+ }
+ function scrollPreparationBinding(params, target) {
+ // A resolved ref wins over coordinates in the actual scroll handler.
+ // Keep the supplied fallback fields only for matching the local request.
+ const ref = params.ref_id && resolveTarget({ ref_id: params.ref_id }) === target ? params.ref_id : null;
+ const point = !ref && Number.isFinite(params.x) && Number.isFinite(params.y) ? { x: params.x, y: params.y } : null;
+ const explicit = !!(params.ref_id || params.selector || point);
+ const pane = target ? scrollPaneFor(target) : null;
+ const direction = params.direction || 'down';
+ const fallbackPane = explicit && target ? scrollFallbackPane(target, direction) : null;
+ const ancestors = new Set();
+ for (let node = target; node; node = bindingParent(node)) ancestors.add(node);
+ ancestors.add(document.scrollingElement || document.documentElement);
+ return { root: document.documentElement, context: JSON.stringify([location.href, document.baseURI, currentViewport()]),
+ request: { ref_id: params.ref_id, selector: params.selector, x: params.x, y: params.y },
+ target, ref, explicit, point, pane, paneRect: rectFor(pane), ancestors,
+ direction, alsoWindow: params.alsoWindow === true, fallbackPane, fallbackRect: rectFor(fallbackPane),
+ invalid: explicit && (!(target instanceof Element) || !target.isConnected || target.ownerDocument !== document) };
+ }
+ function scrollPreparationMatches(op, params = {}) {
+ const binding = op?.scrollIntent;
+ if (!binding || binding.invalid || op.tool !== 'scroll' || (params.kind && params.kind !== 'scroll')
+ || document.documentElement !== binding.root
+ || binding.context !== JSON.stringify([location.href, document.baseURI, currentViewport()])) return false;
+ if (binding.explicit) {
+ if ((binding.ref || op.selector) && (!binding.target?.isConnected || binding.target.ownerDocument !== document)) return false;
+ if (binding.ref && resolveTarget({ ref_id: binding.ref }) !== binding.target) return false;
+ if ((binding.ref || op.selector) && binding.pane && binding.pane !== document.scrollingElement
+ && binding.pane !== document.documentElement && !binding.pane.contains(binding.target)) return false;
+ if (op.selector) {
+ const query = queryOpenTargets(op.selector);
+ if (!query.complete || query.elements.length !== 1 || query.elements[0] !== binding.target) return false;
+ }
+ if (binding.point) {
+ const hit = deepElementFromPoint(binding.point.x, binding.point.y);
+ if (!hit || scrollPaneFor(hit) !== binding.pane || !binding.pane?.isConnected) return false;
+ // Recycled rows within the same pane are fine. Moving or replacing the
+ // pane under an already prepared coordinate is a different recipient.
+ if (!op.dispatched && binding.pane !== document.scrollingElement && binding.pane !== document.documentElement) {
+ if (!scrollPaneRectMatches(binding.pane, binding.paneRect)) return false;
+ }
+ }
+ if (binding.fallbackPane && (!binding.fallbackPane.isConnected
+ || (!op.dispatched && scrollFallbackPane(binding.target, binding.direction) !== binding.fallbackPane)
+ || !scrollPaneRectMatches(binding.fallbackPane, binding.fallbackRect))) return false;
+ if (params.element && !binding.ancestors.has(params.element) && params.element !== binding.fallbackPane) return false;
+ }
+ return !params.element || (params.element instanceof Element && params.element.isConnected && params.element.ownerDocument === document);
+ }
function preparedBindingChanged(op, { allowFocusRebind = false } = {}) {
+ if (op?.scrollIntent) return authorityChanged(op) || !scrollPreparationMatches(op);
if (op?.allowPassiveRebase === true && op.preparedBinding?.model === true && op.focusBound
&& !allowFocusRebind && deepActiveElement() !== op.focusTarget) return true;
return op?.allowPassiveRebase === true && !!op.preparedBinding
@@ -2268,6 +2367,8 @@
&& !operationBindingMatches(op);
}
function nativeReceiverPredicate(op, params) {
+ if (op?.scrollIntent && params.kind && params.kind !== 'scroll') return 'operation_kind_changed';
+ if (op?.scrollIntent && !scrollPreparationMatches(op, params)) return 'scroll_target_changed';
if (op?.preparedBinding?.relaxed && (params.ref_id || params.selector || params.textMatch)
&& resolveTarget(params) !== op.preparedBinding.target) {
revokeRelaxedGrant(); return 'target_changed';
@@ -2298,11 +2399,18 @@
// the previous fence on every uncertain (non-passive) observation.
function authorityChanged(op) {
if (!op || lastUserAt > op.userAt) return true;
- return op.preparedBinding?.model === true
+ return op.scrollIntent || op.preparedBinding?.model === true
? op.preparedInterventionRevision !== interventionRevision || interventionUnreported > 0 || pendingDOM?.source === 'user'
: op.preparedUncertainRevision !== uncertainRevision || uncertainUnreported > 0 || (!!pendingDOM && pendingDOM.source !== 'page');
}
function rebasePassivePreparation(op, params = {}) {
+ if (op?.scrollIntent) {
+ if (!active || op.allowPassiveRebase !== true || op.until < Date.now() || authorityChanged(op)
+ || (params.documentToken && params.documentToken !== documentToken) || !scrollPreparationMatches(op, params)) return false;
+ if (domTimer) flushPendingDOM();
+ op.preparedRevision = revision;
+ return true;
+ }
const relaxed = op?.preparedBinding?.relaxed === true && currentRelaxedGrant(op.preparedBinding.grantToken);
const continuingInput = op?.dispatched && op.kind === 'input' && op.preparedBinding?.model === true;
const continuingPreparation = op?.dispatched && ownedPreparationPhase(op) && op.preparedBinding?.model === true;
@@ -3165,7 +3273,9 @@
const target = resolveTarget(params);
const focusEligible = kindFor(params.tool) === 'input' || modelFocusTools.has(params.tool);
const focusTarget = target || (focusEligible ? deepActiveElement() : null);
- const coordinateSensitive = Number.isFinite(params.x) && Number.isFinite(params.y);
+ const scrollIntent = params.tool === 'scroll' && params.allowPassiveRebase === true && !expected && !inherited
+ ? scrollPreparationBinding(params, target) : null;
+ const coordinateSensitive = !scrollIntent && Number.isFinite(params.x) && Number.isFinite(params.y);
const coordinateHit = coordinateSensitive ? document.elementFromPoint(params.x, params.y) : null;
const coordinateTarget = coordinateSensitive ? target || coordinateHit : null;
const coordinateTargetAtPoint = !target || !coordinateHit || target === coordinateHit || target.contains?.(coordinateHit);
@@ -3192,7 +3302,8 @@
modelBindingInvalid: proofRequired && !certified,
modelBindingReason: proofRequired && !certified ? bindingResult?.reason || 'semantic_binding_changed' : '',
preparedTargetSignature: operationTarget ? signature(operationTarget) : undefined, kinds: new Set(),
- preparedBinding: proofRequired ? certified : allowPassiveRebase && operationTarget ? preparedBinding(operationTarget) : null,
+ scrollIntent,
+ preparedBinding: proofRequired ? certified : !scrollIntent && allowPassiveRebase && operationTarget ? preparedBinding(operationTarget) : null,
preparedInterventionRevision: interventionRevision, preparedUncertainRevision: uncertainRevision, localSynthetic: false,
preparedPageContext: JSON.stringify([location.href, document.baseURI, currentViewport(), window.scrollX, window.scrollY]),
coordinateSensitive, coordinateTarget, coordinateHit, coordinateTargetAtPoint,
@@ -3262,7 +3373,7 @@
const receiverPredicate = nativeReceiverPredicate(op, params);
if (!active || !op || op.modelBindingInvalid || layoutChanged || bindingChanged || (!rebased && (domTimer || unreported))
|| receiverPredicate
- || (params.element && op.target !== params.element)
+ || (params.element && !op.scrollIntent && op.target !== params.element)
|| lastUserAt > op.userAt
|| (Number.isFinite(op.preparedRevision) && op.preparedRevision !== revision)) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
@@ -3341,12 +3452,16 @@
const bindingTool = action === 'ax_prepare_field_for_trusted_type' && params.expectedModelSnapshot
&& currentRelaxedGrant(params.relaxedMonitorGrantToken) ? 'set_field' : action;
const approved = [...operations.values()].reverse().find(op => op.allowPassiveRebase === true && !op.dispatched
- && op.preparedBinding?.target === target && op.until >= Date.now()
- && lastUserAt <= op.userAt
- && (op.preparedBinding.relaxed ? currentRelaxedGrant(op.preparedBinding.grantToken) && operationBindingMatches(op, target)
+ && op.until >= Date.now()
+ && (action === 'scroll' && op.scrollIntent ? !op.finished
+ && ['ref_id', 'selector', 'x', 'y'].every(key => op.scrollIntent.request[key] === params[key])
+ && op.scrollIntent.direction === (params.direction || 'down')
+ && op.scrollIntent.alsoWindow === (params.alsoWindow === true)
+ : lastUserAt <= op.userAt && op.preparedBinding?.target === target && (op.preparedBinding.relaxed ? currentRelaxedGrant(op.preparedBinding.grantToken) && operationBindingMatches(op, target)
: !authorityChanged(op) && (op.preparedBinding.model === true
- ? !modelDecisionPredicate(op.preparedBinding, operationBinding(op, target)) : operationBindingMatches(op, target))));
+ ? !modelDecisionPredicate(op.preparedBinding, operationBinding(op, target)) : operationBindingMatches(op, target)))));
prepare({ selector: params.selector, ref_id: params.ref_id, x: params.x, y: params.y,
+ ...(action === 'scroll' ? { direction: params.direction, alsoWindow: params.alsoWindow } : {}),
textMatch: action === 'click' ? params.text : undefined, tool: bindingTool, operationId, runToken,
...(params.expectedModelSnapshot !== undefined || approved?.expectedModelSnapshot !== undefined
? { expectedModelSnapshot: params.expectedModelSnapshot ?? approved.expectedModelSnapshot } : {}),
@@ -3358,6 +3473,12 @@
// Content-script dispatch is synthetic. It can produce trusted focus,
// scroll and execCommand input, never trusted keys or pointer presses.
if (local) local.localSynthetic = true;
+ if (local && approved?.scrollIntent) {
+ local.scrollIntent = approved.scrollIntent;
+ local.userAt = approved.userAt;
+ local.preparedInterventionRevision = approved.preparedInterventionRevision;
+ local.allowPassiveRebase = true;
+ }
if (local && approved?.preparedBinding?.model === true) {
local.preparedBinding = approved.preparedBinding;
local.modelBindingTool = approved.modelBindingTool || approved.tool;
@@ -3399,7 +3520,7 @@
const layoutChanged = coordinatePreparationShifted(operation);
const bindingChanged = preparedBindingChanged(operation);
const rebased = !layoutChanged && rebasePassivePreparation(operation, { element: target });
- const receiverPredicate = !preparation ? nativeReceiverPredicate(operation, { kind: kind || localOperation.kind }) : '';
+ const receiverPredicate = !preparation ? nativeReceiverPredicate(operation, { kind: kind || localOperation.kind, element: target }) : '';
if (rebased) localOperation.revision = operation.preparedRevision;
if (operation?.modelBindingInvalid || layoutChanged || bindingChanged || receiverPredicate || (!rebased && (domTimer || unreported)) || lastUserAt > localOperation.userAt || revision !== localOperation.revision) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
diff --git a/src/firefox/src/agent/page-feedback-policy.js b/src/firefox/src/agent/page-feedback-policy.js
index a1d4b898d..1129be7f4 100644
--- a/src/firefox/src/agent/page-feedback-policy.js
+++ b/src/firefox/src/agent/page-feedback-policy.js
@@ -1,5 +1,6 @@
/** Tool freshness policy for passive page observations. Mirror in Firefox. */
import { calculateExpression, calculationExpressionSpans, calculationResultInText, tokenizeCalculationExpression } from './calculator.js';
+import { BATCH_TRAILING_OBSERVATION_TOOLS, MAX_BATCH_ACTIONS } from './browser-batch.js';
const FRESH_READS = new Set([
'get_accessibility_tree', 'get_interactive_elements', 'read_page', 'read_page_source',
@@ -255,6 +256,31 @@ function axTargetContext(page, refId) {
export function pageFeedbackCallPolicy(name, args = {}, state = {}, currentPage = null, events = []) {
if (FRESH_READS.has(name)) return { kind: 'observe' };
if (DETACHED_TOOLS.has(name)) return { kind: 'independent' };
+ // Scrolling explores the current document. A counter or animation changing
+ // while the model thinks does not invalidate that intent. The content monitor
+ // resolves its live pane and checks intervention, viewport and target identity
+ // again at dispatch; this does not grant any click or input action.
+ if (name === 'scroll' && (args.frameId == null || args.frameId === 0)
+ && (args.targetFrame == null || args.targetFrame === 0)
+ && !args.allFrames && !args.all && !args.urlFilter) return { kind: 'scroll' };
+ // The parent is checked before expansion. Retain only scrolling followed by
+ // ordinary observations; each child still passes the normal batch validator
+ // and its own preparation/dispatch checks. Never retain an arbitrary batch.
+ if (name === 'browser_batch' && Array.isArray(args.actions) && args.actions.length > 0
+ && args.actions.length <= MAX_BATCH_ACTIONS.full) {
+ let observed = false, scrolled = false;
+ const scrollBatch = args.actions.every(action => {
+ if (!action || typeof action !== 'object' || Array.isArray(action)
+ || Object.keys(action).some(key => !['name', 'input'].includes(key))
+ || !action.input || typeof action.input !== 'object' || Array.isArray(action.input)) return false;
+ if (BATCH_TRAILING_OBSERVATION_TOOLS.has(action.name)) { observed = true; return true; }
+ if (observed || action.name !== 'scroll'
+ || pageFeedbackCallPolicy('scroll', action.input).kind !== 'scroll') return false;
+ scrolled = true;
+ return true;
+ });
+ if (scrollBatch && scrolled) return { kind: 'scroll' };
+ }
if (['fetch_url', 'research_url'].includes(name) && httpUrl(args.url)
&& ['GET', 'HEAD'].includes(String(args.method || 'GET').toUpperCase())
&& !args.replayRequestId && !args.apiReplayRequestId && !args.body) return { kind: 'independent' };
diff --git a/src/firefox/src/agent/page-feedback-recovery.js b/src/firefox/src/agent/page-feedback-recovery.js
index 5586fa225..640c9811d 100644
--- a/src/firefox/src/agent/page-feedback-recovery.js
+++ b/src/firefox/src/agent/page-feedback-recovery.js
@@ -5,8 +5,8 @@ export const PAGE_FEEDBACK_NUDGE_AT = 2;
export const PAGE_FEEDBACK_STOP_AT = 5;
export const PAGE_FEEDBACK_STOP_CODE = 'page_unstable';
-const STOP_MESSAGE = 'Stopped because the page kept changing before the next action could be safely dispatched. '
- + 'The task is incomplete; no stale action was dispatched. Wait for the page to settle, then choose Retry.';
+const STOP_MESSAGE = 'WebBrain could not validate the next action after repeated browser updates. '
+ + 'The task is incomplete; no stale action was dispatched. Choose Retry to inspect the current page and continue.';
const NUDGE_MESSAGE = '[PAGE KEEPS CHANGING: Browser updates repeatedly prevented the next action from being dispatched. '
+ 'Use the fresh observation to choose an action whose intended target can be verified. '
+ 'Do not repeat stale targets or coordinates.]';
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index 6d6231624..fec5127b9 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -221,7 +221,8 @@ function preparationVetoDiagnostics(veto) {
if (new Set(['semantic_binding_changed', 'geometry_changed', 'model_snapshot_invalid', 'model_snapshot_expired',
'focus_changed', 'pending_feedback', 'revision_changed', 'document_changed', 'document_revision_changed',
'intervention_changed', 'target_changed', 'target_hit_changed', 'operation_expired', 'operation_missing',
- 'run_ended', 'target_unresolved', 'target_uncovered', 'current_owner_changed']).has(veto.predicate)) facts.predicate = veto.predicate;
+ 'run_ended', 'target_unresolved', 'target_uncovered', 'current_owner_changed',
+ 'operation_kind_changed', 'scroll_target_changed']).has(veto.predicate)) facts.predicate = veto.predicate;
return facts;
}
@@ -343,6 +344,10 @@ function safeAction(tool, args = {}) {
// click({text}) is a target label, never typed text or a field value.
if (['click', 'iframe_click'].includes(tool) && typeof args.text === 'string') target.textMatch = args.text;
for (const key of ['x', 'y']) if (Number.isFinite(args[key])) target[key] = args[key];
+ if (tool === 'scroll') {
+ if (typeof args.direction === 'string') target.direction = args.direction;
+ if (typeof args.alsoWindow === 'boolean') target.alsoWindow = args.alsoWindow;
+ }
return { tool, ...target };
}
@@ -1165,6 +1170,10 @@ export const pageFeedbackMethods = {
_pageFeedbackBlocksCall(tabId, name, args = {}) {
const run = this._pageFeedbackRuns?.get(tabId);
+ if (name === 'scroll' && run?.modelState
+ && (this._hasPendingSteering?.(tabId)
+ || !this._pageFeedbackStateCurrent(tabId, run, run.modelState, run.url, { rootOnly: true })
+ || (run.dispatchCall?.name === name && run.dispatchCall.modelState !== run.modelState))) return true;
if (run?.dispatchCall?.expectedModelSnapshot
&& (run.modelState !== run.dispatchCall.modelState
|| !this._pageFeedbackStateCurrent(tabId, run, run.dispatchCall.modelState))) return true;
@@ -1173,12 +1182,12 @@ export const pageFeedbackMethods = {
const policy = pageFeedbackCallPolicy(name, args, run.modelState, run.latestPage, events);
// Automatic churn (a superset of relaxed-retry DOM) revalidates per policy.
if (!isAutomaticPageFeedback(events) || !this._pageFeedbackStateCurrent(tabId, run, run.modelState, run.url,
- { rootOnly: policy.kind === 'request' })) return true;
+ { rootOnly: ['request', 'scroll'].includes(policy.kind) })) return true;
// Unknown DOM is observed while an agent action is in flight. It is that
// call's own uncertain effect only after the call sent input (typeahead
// renders between keys). Observed during an earlier action, or during this
// call's preparation, it stays a fence: re-observe before new input.
- if (!this._monitoringRetryCurrent(tabId, run) && events.some(event => (unknownAttribution.get(event) || [])
+ if (policy.kind !== 'scroll' && !this._monitoringRetryCurrent(tabId, run) && events.some(event => (unknownAttribution.get(event) || [])
.some(({ call, afterInput }) => call && call.mutation !== false && !(call === run.dispatchCall && afterInput)))) return true;
const policyEvents = this._monitoringRetryCurrent(tabId, run)
? events.map(event => event.source === 'unknown' && event.kind === 'dom' ? { ...event, source: 'page' } : event) : events;
@@ -1353,12 +1362,12 @@ export const pageFeedbackMethods = {
documentMatches = state.documentId ? documentId === state.documentId : state.frames.has(0);
}
const currentRun = this._pageFeedbackRuns.get(tabId) === run;
- const concreteRequests = !!responseToolCalls?.length && responseToolCalls.every(call => {
- try { return pageFeedbackCallPolicy(call.function?.name, JSON.parse(call.function.arguments)).kind === 'request'; }
+ const rootScopedCalls = !!responseToolCalls?.length && responseToolCalls.every(call => {
+ try { return ['request', 'scroll'].includes(pageFeedbackCallPolicy(call.function?.name, JSON.parse(call.function.arguments)).kind); }
catch { return false; }
});
const stateCurrent = documentMatches && currentRun
- ? this._pageFeedbackStateCurrent(tabId, run, state, url, { rootOnly: concreteRequests }) : null;
+ ? this._pageFeedbackStateCurrent(tabId, run, state, url, { rootOnly: rootScopedCalls }) : null;
const sameDocument = documentMatches && currentRun && stateCurrent;
// A tool-free Ask answer or a strictly owned literal arithmetic-only task
// is independent of passive churn. Task-only scope permits normal protocol
@@ -1500,10 +1509,19 @@ export function installPageFeedback(Agent) {
const execute = Agent.prototype.executeTool;
Agent.prototype.executeTool = async function(tabId, name, args, onUpdate, executionContext) {
const run = this._pageFeedbackRuns?.get(tabId);
+ // Resolve the browser's last-interaction hint before monitor preparation so
+ // the background and local scroll bind the same requested pane.
+ if (name === 'scroll' && this._augmentScrollArgsWithLastInteraction) {
+ args = await this._augmentScrollArgsWithLastInteraction(tabId, args);
+ }
const mutation = this.constructor.STATE_CHANGE_TOOLS.has(name) || ['upload_file', 'solve_captcha', 'apply_captcha_solution'].includes(name);
// A model cannot supply the private expected binding. Even without queued
// feedback, use the identity captured before inference for a focused download.
const modelState = run?.modelState;
+ const scrollStateCurrent = () => this._pageFeedbackRuns?.get(tabId) === run
+ && run.modelState === modelState && !this._hasPendingSteering?.(tabId)
+ && this._pageFeedbackStateCurrent(tabId, run, modelState, run.url, { rootOnly: true });
+ if (run && modelState && name === 'scroll' && !scrollStateCurrent()) return pageFeedbackPendingResult();
const batchId = executionContext?._browserBatch?.parentId;
const policy = pageFeedbackCallPolicy(name, args || {}, modelState, run?.latestPage);
if (run && policy.kind === 'media_resolve') return this._resolvePageFeedbackMedia(tabId, policy.target);
@@ -1521,8 +1539,8 @@ export function installPageFeedback(Agent) {
modelBound = validated === true;
}
const targetValidated = !!run && run.validatedTargets?.get(`${name}:${JSON.stringify(args)}`) === run.revision;
- const allowPassiveRebase = modelBound || targetValidated || (policy.kind === 'navigate'
- && !!run && this._pageFeedbackStateCurrent(tabId, run, run.modelState));
+ const allowPassiveRebase = modelBound || targetValidated || (['navigate', 'scroll'].includes(policy.kind)
+ && !!run && this._pageFeedbackStateCurrent(tabId, run, run.modelState, run.url, { rootOnly: policy.kind === 'scroll' }));
if (run && mutation && !modelBound && this._pageFeedbackBlocksCall(tabId, name, args)) return pageFeedbackPendingResult();
const dispatchState = executionContext?._contentActionDispatchState || { started: false };
executionContext = { ...executionContext, _contentActionDispatchState: dispatchState,
@@ -1560,10 +1578,12 @@ export function installPageFeedback(Agent) {
if ((modelBound ? prepared?.ready === true && prepared.modelBindingValid === true
&& this._pageFeedbackRuns?.get(tabId) === run && run.modelState === modelState
&& this._pageFeedbackStateCurrent(tabId, run, modelState)
- : run.revision === revision || policy.kind === 'navigate') && allowPassiveRebase) run.dispatchCall.allowPassiveRebase = true;
+ : policy.kind === 'scroll' ? scrollStateCurrent()
+ : run.revision === revision || policy.kind === 'navigate') && allowPassiveRebase) run.dispatchCall.allowPassiveRebase = true;
}
try {
if (modelBound && !run.dispatchCall.allowPassiveRebase) { noDispatch = true; return pageFeedbackPendingResult(); }
+ if (run && modelState && name === 'scroll' && !scrollStateCurrent()) { noDispatch = true; return pageFeedbackPendingResult(); }
if (run && mutation && this._pageFeedbackBlocksCall(tabId, name, args)) return pageFeedbackPendingResult();
const observationDocument = run?.frames.get(0)?.token, observationUrl = run?.url;
// Completion verification reads the page through this wrapper. Neither
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 6be47beb7..8bcd74468 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -3685,7 +3685,7 @@
if (target) {
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target });
dispatched = true;
if (direction === 'down') target.scrollBy(0, amount);
else if (direction === 'up') target.scrollBy(0, -amount);
@@ -3696,7 +3696,7 @@
// Always also scroll the window in case both are needed.
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: document.scrollingElement || document.documentElement });
dispatched = true;
if (direction === 'down') window.scrollBy(0, amount);
else if (direction === 'up') window.scrollBy(0, -amount);
@@ -3885,7 +3885,7 @@
if (target) {
containerBefore = target.scrollTop;
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target });
dispatched = true;
scrollElementInstant(target, direction, amount);
containerAfter = target.scrollTop;
@@ -3896,7 +3896,7 @@
const shouldScrollWindow = params.alsoWindow === true || !movedContainer;
if (shouldScrollWindow && windowCanMove) {
if (actionDeadlineExpired()) return deadlineFailure();
- window.__wbPageMonitor?.beforeLocalDispatch();
+ window.__wbPageMonitor?.beforeLocalDispatch({ kind: 'scroll', target: scrollingElement });
dispatched = true;
if (direction === 'down') window.scrollBy(0, amount);
else if (direction === 'up') window.scrollBy(0, -amount);
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 8e4cc2e81..941f04716 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -2260,7 +2260,106 @@
function ownedPreparationPhase(op) {
return op?.pointerPreparationOnly === true || op?.typingPreparationOnly === true || op?.selectorPreparationOnly === true;
}
+ // Scrolling is a relative view operation. Feed contents and uncertain DOM
+ // attribution are not its authority: the current document, human ownership
+ // and requested scroll pane are. Keep this proof separate from certificates
+ // for clicks and input, which must still bind the observed page semantics.
+ function scrollPaneFor(origin) {
+ const documentScrollable = Math.max(document.body?.scrollHeight || 0, document.documentElement?.scrollHeight || 0,
+ document.scrollingElement?.scrollHeight || 0) > window.innerHeight + 10;
+ const skipOrigin = /^(INPUT|TEXTAREA|SELECT)$/.test(origin?.tagName || '') || origin?.isContentEditable;
+ for (let node = skipOrigin ? bindingParent(origin) : origin; node && node !== document.body && node !== document.documentElement; node = bindingParent(node)) {
+ const style = getComputedStyle(node);
+ if (node.scrollHeight > node.clientHeight + 10 && (/^(auto|scroll|overlay)$/.test(style.overflowY)
+ || (!documentScrollable && style.overflowY === 'hidden'))) return node;
+ }
+ return document.scrollingElement || document.documentElement;
+ }
+ // Match smartScrollPage's last resort when neither the origin's ancestors
+ // nor the window can move. Capture that recipient before dispatch rather
+ // than granting arbitrary containers permission to borrow the origin.
+ function scrollFallbackPane(origin, direction) {
+ const documentHeight = Math.max(document.body?.scrollHeight || 0, document.documentElement?.scrollHeight || 0,
+ document.scrollingElement?.scrollHeight || 0);
+ const allowHidden = documentHeight <= window.innerHeight + 10;
+ const canMove = (position, maximum) => direction === 'down' || direction === 'bottom' ? position < maximum - 1
+ : direction === 'up' || direction === 'top' ? position > 1 : false;
+ const scrollable = node => {
+ if (!node || node === document.body || node === document.documentElement
+ || node.scrollHeight <= node.clientHeight + 10) return false;
+ const overflow = getComputedStyle(node).overflowY;
+ return (/^(auto|scroll|overlay)$/.test(overflow) || (allowHidden && overflow === 'hidden'))
+ && canMove(node.scrollTop, Math.max(0, node.scrollHeight - node.clientHeight));
+ };
+ const skipOrigin = /^(INPUT|TEXTAREA|SELECT)$/.test(origin?.tagName || '') || origin?.isContentEditable;
+ for (let node = skipOrigin ? origin.parentElement : origin; node && node !== document.body && node !== document.documentElement; node = node.parentElement) {
+ if (scrollable(node)) return null;
+ }
+ if (canMove(window.scrollY, Math.max(0, documentHeight - window.innerHeight))) return null;
+ let best = null, bestArea = 0;
+ for (const node of document.querySelectorAll('div, section, main, article, aside, [role="main"], [role="dialog"], [role="region"], [role="listbox"], [role="menu"]')) {
+ if (!scrollable(node)) continue;
+ const rect = node.getBoundingClientRect();
+ if (rect.width < 1 || rect.height < 1 || rect.bottom < 0 || rect.top > window.innerHeight || rect.right < 0 || rect.left > window.innerWidth) continue;
+ const area = rect.width * rect.height;
+ if (area > bestArea) { best = node; bestArea = area; }
+ }
+ return bestArea > window.innerWidth * window.innerHeight * 0.15 ? best : null;
+ }
+ function scrollPaneRectMatches(pane, before) {
+ const current = rectFor(pane);
+ return !!current && !!before && ['x', 'y', 'width', 'height'].every(key => Math.abs(current[key] - before[key]) <= 2);
+ }
+ function scrollPreparationBinding(params, target) {
+ // A resolved ref wins over coordinates in the actual scroll handler.
+ // Keep the supplied fallback fields only for matching the local request.
+ const ref = params.ref_id && resolveTarget({ ref_id: params.ref_id }) === target ? params.ref_id : null;
+ const point = !ref && Number.isFinite(params.x) && Number.isFinite(params.y) ? { x: params.x, y: params.y } : null;
+ const explicit = !!(params.ref_id || params.selector || point);
+ const pane = target ? scrollPaneFor(target) : null;
+ const direction = params.direction || 'down';
+ const fallbackPane = explicit && target ? scrollFallbackPane(target, direction) : null;
+ const ancestors = new Set();
+ for (let node = target; node; node = bindingParent(node)) ancestors.add(node);
+ ancestors.add(document.scrollingElement || document.documentElement);
+ return { root: document.documentElement, context: JSON.stringify([location.href, document.baseURI, currentViewport()]),
+ request: { ref_id: params.ref_id, selector: params.selector, x: params.x, y: params.y },
+ target, ref, explicit, point, pane, paneRect: rectFor(pane), ancestors,
+ direction, alsoWindow: params.alsoWindow === true, fallbackPane, fallbackRect: rectFor(fallbackPane),
+ invalid: explicit && (!(target instanceof Element) || !target.isConnected || target.ownerDocument !== document) };
+ }
+ function scrollPreparationMatches(op, params = {}) {
+ const binding = op?.scrollIntent;
+ if (!binding || binding.invalid || op.tool !== 'scroll' || (params.kind && params.kind !== 'scroll')
+ || document.documentElement !== binding.root
+ || binding.context !== JSON.stringify([location.href, document.baseURI, currentViewport()])) return false;
+ if (binding.explicit) {
+ if ((binding.ref || op.selector) && (!binding.target?.isConnected || binding.target.ownerDocument !== document)) return false;
+ if (binding.ref && resolveTarget({ ref_id: binding.ref }) !== binding.target) return false;
+ if ((binding.ref || op.selector) && binding.pane && binding.pane !== document.scrollingElement
+ && binding.pane !== document.documentElement && !binding.pane.contains(binding.target)) return false;
+ if (op.selector) {
+ const query = queryOpenTargets(op.selector);
+ if (!query.complete || query.elements.length !== 1 || query.elements[0] !== binding.target) return false;
+ }
+ if (binding.point) {
+ const hit = deepElementFromPoint(binding.point.x, binding.point.y);
+ if (!hit || scrollPaneFor(hit) !== binding.pane || !binding.pane?.isConnected) return false;
+ // Recycled rows within the same pane are fine. Moving or replacing the
+ // pane under an already prepared coordinate is a different recipient.
+ if (!op.dispatched && binding.pane !== document.scrollingElement && binding.pane !== document.documentElement) {
+ if (!scrollPaneRectMatches(binding.pane, binding.paneRect)) return false;
+ }
+ }
+ if (binding.fallbackPane && (!binding.fallbackPane.isConnected
+ || (!op.dispatched && scrollFallbackPane(binding.target, binding.direction) !== binding.fallbackPane)
+ || !scrollPaneRectMatches(binding.fallbackPane, binding.fallbackRect))) return false;
+ if (params.element && !binding.ancestors.has(params.element) && params.element !== binding.fallbackPane) return false;
+ }
+ return !params.element || (params.element instanceof Element && params.element.isConnected && params.element.ownerDocument === document);
+ }
function preparedBindingChanged(op, { allowFocusRebind = false } = {}) {
+ if (op?.scrollIntent) return authorityChanged(op) || !scrollPreparationMatches(op);
if (op?.allowPassiveRebase === true && op.preparedBinding?.model === true && op.focusBound
&& !allowFocusRebind && deepActiveElement() !== op.focusTarget) return true;
return op?.allowPassiveRebase === true && !!op.preparedBinding
@@ -2268,6 +2367,8 @@
&& !operationBindingMatches(op);
}
function nativeReceiverPredicate(op, params) {
+ if (op?.scrollIntent && params.kind && params.kind !== 'scroll') return 'operation_kind_changed';
+ if (op?.scrollIntent && !scrollPreparationMatches(op, params)) return 'scroll_target_changed';
if (op?.preparedBinding?.relaxed && (params.ref_id || params.selector || params.textMatch)
&& resolveTarget(params) !== op.preparedBinding.target) {
revokeRelaxedGrant(); return 'target_changed';
@@ -2298,11 +2399,18 @@
// the previous fence on every uncertain (non-passive) observation.
function authorityChanged(op) {
if (!op || lastUserAt > op.userAt) return true;
- return op.preparedBinding?.model === true
+ return op.scrollIntent || op.preparedBinding?.model === true
? op.preparedInterventionRevision !== interventionRevision || interventionUnreported > 0 || pendingDOM?.source === 'user'
: op.preparedUncertainRevision !== uncertainRevision || uncertainUnreported > 0 || (!!pendingDOM && pendingDOM.source !== 'page');
}
function rebasePassivePreparation(op, params = {}) {
+ if (op?.scrollIntent) {
+ if (!active || op.allowPassiveRebase !== true || op.until < Date.now() || authorityChanged(op)
+ || (params.documentToken && params.documentToken !== documentToken) || !scrollPreparationMatches(op, params)) return false;
+ if (domTimer) flushPendingDOM();
+ op.preparedRevision = revision;
+ return true;
+ }
const relaxed = op?.preparedBinding?.relaxed === true && currentRelaxedGrant(op.preparedBinding.grantToken);
const continuingInput = op?.dispatched && op.kind === 'input' && op.preparedBinding?.model === true;
const continuingPreparation = op?.dispatched && ownedPreparationPhase(op) && op.preparedBinding?.model === true;
@@ -3165,7 +3273,9 @@
const target = resolveTarget(params);
const focusEligible = kindFor(params.tool) === 'input' || modelFocusTools.has(params.tool);
const focusTarget = target || (focusEligible ? deepActiveElement() : null);
- const coordinateSensitive = Number.isFinite(params.x) && Number.isFinite(params.y);
+ const scrollIntent = params.tool === 'scroll' && params.allowPassiveRebase === true && !expected && !inherited
+ ? scrollPreparationBinding(params, target) : null;
+ const coordinateSensitive = !scrollIntent && Number.isFinite(params.x) && Number.isFinite(params.y);
const coordinateHit = coordinateSensitive ? document.elementFromPoint(params.x, params.y) : null;
const coordinateTarget = coordinateSensitive ? target || coordinateHit : null;
const coordinateTargetAtPoint = !target || !coordinateHit || target === coordinateHit || target.contains?.(coordinateHit);
@@ -3192,7 +3302,8 @@
modelBindingInvalid: proofRequired && !certified,
modelBindingReason: proofRequired && !certified ? bindingResult?.reason || 'semantic_binding_changed' : '',
preparedTargetSignature: operationTarget ? signature(operationTarget) : undefined, kinds: new Set(),
- preparedBinding: proofRequired ? certified : allowPassiveRebase && operationTarget ? preparedBinding(operationTarget) : null,
+ scrollIntent,
+ preparedBinding: proofRequired ? certified : !scrollIntent && allowPassiveRebase && operationTarget ? preparedBinding(operationTarget) : null,
preparedInterventionRevision: interventionRevision, preparedUncertainRevision: uncertainRevision, localSynthetic: false,
preparedPageContext: JSON.stringify([location.href, document.baseURI, currentViewport(), window.scrollX, window.scrollY]),
coordinateSensitive, coordinateTarget, coordinateHit, coordinateTargetAtPoint,
@@ -3262,7 +3373,7 @@
const receiverPredicate = nativeReceiverPredicate(op, params);
if (!active || !op || op.modelBindingInvalid || layoutChanged || bindingChanged || (!rebased && (domTimer || unreported))
|| receiverPredicate
- || (params.element && op.target !== params.element)
+ || (params.element && !op.scrollIntent && op.target !== params.element)
|| lastUserAt > op.userAt
|| (Number.isFinite(op.preparedRevision) && op.preparedRevision !== revision)) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
@@ -3341,12 +3452,16 @@
const bindingTool = action === 'ax_prepare_field_for_trusted_type' && params.expectedModelSnapshot
&& currentRelaxedGrant(params.relaxedMonitorGrantToken) ? 'set_field' : action;
const approved = [...operations.values()].reverse().find(op => op.allowPassiveRebase === true && !op.dispatched
- && op.preparedBinding?.target === target && op.until >= Date.now()
- && lastUserAt <= op.userAt
- && (op.preparedBinding.relaxed ? currentRelaxedGrant(op.preparedBinding.grantToken) && operationBindingMatches(op, target)
+ && op.until >= Date.now()
+ && (action === 'scroll' && op.scrollIntent ? !op.finished
+ && ['ref_id', 'selector', 'x', 'y'].every(key => op.scrollIntent.request[key] === params[key])
+ && op.scrollIntent.direction === (params.direction || 'down')
+ && op.scrollIntent.alsoWindow === (params.alsoWindow === true)
+ : lastUserAt <= op.userAt && op.preparedBinding?.target === target && (op.preparedBinding.relaxed ? currentRelaxedGrant(op.preparedBinding.grantToken) && operationBindingMatches(op, target)
: !authorityChanged(op) && (op.preparedBinding.model === true
- ? !modelDecisionPredicate(op.preparedBinding, operationBinding(op, target)) : operationBindingMatches(op, target))));
+ ? !modelDecisionPredicate(op.preparedBinding, operationBinding(op, target)) : operationBindingMatches(op, target)))));
prepare({ selector: params.selector, ref_id: params.ref_id, x: params.x, y: params.y,
+ ...(action === 'scroll' ? { direction: params.direction, alsoWindow: params.alsoWindow } : {}),
textMatch: action === 'click' ? params.text : undefined, tool: bindingTool, operationId, runToken,
...(params.expectedModelSnapshot !== undefined || approved?.expectedModelSnapshot !== undefined
? { expectedModelSnapshot: params.expectedModelSnapshot ?? approved.expectedModelSnapshot } : {}),
@@ -3358,6 +3473,12 @@
// Content-script dispatch is synthetic. It can produce trusted focus,
// scroll and execCommand input, never trusted keys or pointer presses.
if (local) local.localSynthetic = true;
+ if (local && approved?.scrollIntent) {
+ local.scrollIntent = approved.scrollIntent;
+ local.userAt = approved.userAt;
+ local.preparedInterventionRevision = approved.preparedInterventionRevision;
+ local.allowPassiveRebase = true;
+ }
if (local && approved?.preparedBinding?.model === true) {
local.preparedBinding = approved.preparedBinding;
local.modelBindingTool = approved.modelBindingTool || approved.tool;
@@ -3399,7 +3520,7 @@
const layoutChanged = coordinatePreparationShifted(operation);
const bindingChanged = preparedBindingChanged(operation);
const rebased = !layoutChanged && rebasePassivePreparation(operation, { element: target });
- const receiverPredicate = !preparation ? nativeReceiverPredicate(operation, { kind: kind || localOperation.kind }) : '';
+ const receiverPredicate = !preparation ? nativeReceiverPredicate(operation, { kind: kind || localOperation.kind, element: target }) : '';
if (rebased) localOperation.revision = operation.preparedRevision;
if (operation?.modelBindingInvalid || layoutChanged || bindingChanged || receiverPredicate || (!rebased && (domTimer || unreported)) || lastUserAt > localOperation.userAt || revision !== localOperation.revision) {
const error = new Error('Browser changed during action preparation. Re-observe before acting.');
diff --git a/test/ask-completion-churn.mjs b/test/ask-completion-churn.mjs
index 778840d65..4324df214 100644
--- a/test/ask-completion-churn.mjs
+++ b/test/ask-completion-churn.mjs
@@ -208,7 +208,7 @@ for (const build of ['chrome', 'firefox']) {
test(`${build}: automatic iframe churn has a finite retry budget for dependent answers`, async () => {
const f = fixture(Agent, { streaming: false, response: 'done', change: 'dependent-continuous', dependent: true });
try {
- assert.match(await f.run(), /page kept changing/i);
+ assert.match(await f.run(), /could not validate the next action after repeated browser updates/i);
assert.deepEqual(f.dispatched, []);
assert.deepEqual(f.counts(), { mainCalls: 5, scopeCalls: 2 });
assert.ok(f.updates.some(update => update.type === 'error' && update.data.code === 'page_unstable'));
@@ -239,7 +239,7 @@ for (const build of ['chrome', 'firefox']) {
test(`${build}: a foreign live monitor cannot grant an independent answer proof`, async () => {
const f = fixture(Agent, { streaming: false, response: 'done', change: 'foreign-monitor' });
try {
- assert.match(await f.run(), /page kept changing/i);
+ assert.match(await f.run(), /could not validate the next action after repeated browser updates/i);
assert.deepEqual(f.dispatched, []);
assert.deepEqual(f.counts(), { mainCalls: 5, scopeCalls: 0 });
} finally { f.clean(); }
diff --git a/test/feedback-automatic-churn.mjs b/test/feedback-automatic-churn.mjs
index a850c599f..f577a91cb 100644
--- a/test/feedback-automatic-churn.mjs
+++ b/test/feedback-automatic-churn.mjs
@@ -17,20 +17,25 @@ const api = {
sendMessage: async (id, message) => {
const page = pages.get(id) || {};
if (message.action === 'page_monitor_capture_model') {
- return { ready: true, runToken: message.params.runToken, documentToken: 'main-token', snapshotToken: 'snapshot-' + (++snapshots),
- targetCount: 2, focusedTargetAvailable: false, page: { success: true, url: PAGE_URL, pageContent: TREE } };
+ return { ready: true, runToken: message.params.runToken, documentToken: page.documentToken ?? 'main-token', snapshotToken: 'snapshot-' + (++snapshots),
+ targetCount: 2, focusedTargetAvailable: false, page: { success: true, url: PAGE_URL, pageContent: page.tree ?? TREE } };
}
if (message.action === 'page_monitor_validate_model') {
(page.validations ??= []).push(message.params.snapshotToken);
const changed = page.targetChanged === true;
return { ready: !changed, ...(changed ? { reason: 'target_changed' } : {}), runToken: message.params.runToken,
- documentToken: 'main-token', snapshotToken: message.params.snapshotToken };
+ documentToken: page.documentToken ?? 'main-token', snapshotToken: message.params.snapshotToken };
+ }
+ if (message.action === 'page_monitor_prepare') {
+ (page.preparations ??= []).push(message.params);
+ page.onPrepare?.(message.params);
+ return { ready: true, modelBindingValid: true };
}
- if (message.action === 'page_monitor_prepare') return { ready: true, modelBindingValid: true };
return { ready: true };
},
},
- webNavigation: { getAllFrames: async () => [{ frameId: 0, parentFrameId: -1, documentId: 'main-document', url: PAGE_URL },
+ webNavigation: { getAllFrames: async ({ tabId }) => [{ frameId: 0, parentFrameId: -1,
+ documentId: pages.get(tabId)?.documentId ?? 'main-document', url: PAGE_URL },
{ frameId: 1, parentFrameId: 0, documentId: 'ad-document', url: 'https://ads.example/slot' }] },
scripting: { executeScript: async () => [{ result: null }] },
};
@@ -77,7 +82,9 @@ function bindFrames(agent, tab) {
return value => assert.equal(agent.observePageFeedback(from, { ...state, seq: ++seq, revision: seq,
kind: 'dom', source: 'page', target: 'span#price', ...value }).accepted, true);
};
- return { main: make(0, 'main-document', 'main-token', PAGE_URL), child: make(1, 'ad-document', 'ad-token', 'https://ads.example/slot') };
+ const page = pages.get(tab) || {};
+ return { main: make(0, page.documentId ?? 'main-document', page.documentToken ?? 'main-token', PAGE_URL),
+ child: make(1, 'ad-document', 'ad-token', 'https://ads.example/slot') };
}
const call = (id, name, args) => ({ toolCalls: [{ id, function: { name, arguments: JSON.stringify(args) } }] });
@@ -95,6 +102,7 @@ const AUTOMATIC = {
for (const build of ['chrome', 'firefox']) {
const { Agent } = await import(`../src/${build}/src/agent/agent.js`);
const policy = await import(`../src/${build}/src/agent/page-feedback-policy.js`);
+ const { beforePageAgentDispatch } = await import(`../src/${build}/src/agent/page-feedback.js`);
test(`${build}: automatic classifier separates page churn from intervention and unknown navigation`, () => {
const automatic = [
@@ -164,6 +172,259 @@ for (const build of ['chrome', 'firefox']) {
}
});
+ test(`${build}: scroll intent survives automatic text changes without granting mixed batches or intervention a bypass`, () => {
+ const page = { success: true, pageContent: TREE };
+ const state = { url: PAGE_URL, page };
+ const changed = { ...page, pageContent: TREE + '\n status "Updated moments ago" [ref_clock]' };
+ const automatic = [
+ { kind: 'dom', source: 'page', frameId: 0, target: 'span#clock' },
+ { kind: 'dom', source: 'unknown', frameId: 0, target: 'div#feed' },
+ ];
+ const retains = (name, args, events = automatic) => policy.canRetainPageFeedbackCalls(
+ [{ function: { name, arguments: JSON.stringify(args) } }], state, changed, events);
+ const inputs = [{ direction: 'down', amount: 600 }, { direction: 'bottom', ref_id: 'ref_main' },
+ { direction: 'down', x: 320, y: 500 }];
+ for (const input of inputs) {
+ assert.equal(policy.pageFeedbackCallPolicy('scroll', input, state, changed, automatic).kind, 'scroll');
+ assert.equal(retains('scroll', input), true, 'Unrelated fresh AX text must not invalidate scrolling');
+ assert.equal(retains('scroll', input, [{ kind: 'dom', source: 'page', frameId: 0, target: 'span#clock' }]), true);
+ for (const event of [{ kind: 'scroll', source: 'user', frameId: 0, target: 'html' },
+ { kind: 'navigation', source: 'unknown', frameId: 0 }]) {
+ assert.equal(retains('scroll', input, [...automatic, event]), false);
+ }
+ }
+ const batch = { actions: inputs.map(input => ({ name: 'scroll', input })) };
+ assert.equal(policy.pageFeedbackCallPolicy('browser_batch', batch, state, changed, automatic).kind, 'scroll');
+ assert.equal(retains('browser_batch', batch), true, 'Scroll-only batches use the same freshness policy');
+ for (const action of [{ name: 'click', input: { x: 320, y: 500 } },
+ { name: 'execute_js', input: { code: 'save()' } }]) {
+ assert.equal(retains('browser_batch', { actions: [...batch.actions, action] }), false,
+ 'Adding a fenced action must not inherit scroll retention');
+ }
+ for (const input of [{ direction: 'down', frameId: 2 }, { direction: 'down', targetFrame: 2 },
+ { direction: 'down', allFrames: true }]) assert.equal(retains('scroll', input), false,
+ 'Only the current top document has this scroll policy');
+ assert.equal(retains('browser_batch', { actions: [] }), false);
+ assert.equal(retains('browser_batch', { actions: [{ name: 'browser_batch', input: batch }] }), false);
+ });
+
+ for (const streaming of [false, true]) {
+ for (const scenario of [
+ { name: 'viewport scroll', input: { direction: 'down', amount: 600 } },
+ { name: 'pane ref scroll', input: { direction: 'bottom', ref_id: 'ref_main' } },
+ { name: 'batched coordinate scroll', input: { direction: 'down', x: 320, y: 500 }, batch: true },
+ ]) {
+ test(`${build}: ${streaming ? 'stream' : 'chat'} ${scenario.name} executes once through continuous DOM and AX text updates`, async () => {
+ const tab = nextTab++, dispatched = [], updates = [];
+ let agent, frames, requests = 0, terminal = null;
+ pages.set(tab, {});
+ agent = makeAgent(Agent, async () => {
+ requests++;
+ assert.ok(requests <= 2, 'Automatic changes must not regenerate the same scroll or completed answer');
+ pages.get(tab).tree = TREE + `\n status "Refresh ${requests}" [ref_clock]`;
+ // Each model call overlaps real main-document updates, including
+ // unattributed effects. The fresh AX tree changes as well.
+ frames.main({ kind: 'dom', source: 'page', target: 'span#clock' });
+ frames.main({ kind: 'dom', source: 'unknown', target: 'div#feed' });
+ AUTOMATIC.child_navigation(agent, tab, frames);
+ if (requests > 1) return call('finished', 'done', { summary: 'Scrolled down', outcome: 'success' });
+ return scenario.batch
+ ? call('scroll-batch', 'browser_batch', { actions: [{ name: 'scroll', input: scenario.input }] })
+ : call('scroll-once', 'scroll', scenario.input);
+ });
+ agent.maxSteps = 7;
+ agent._endTraceRun = async (_tab, _run, status) => { terminal = status; };
+ agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => { frames = bindFrames(agent, tab); return { role: 'user', content }; };
+ pages.get(tab).onPrepare = params => {
+ if (params.tool === 'scroll') frames.main({ kind: 'dom', source: 'unknown', target: 'div#during-prepare' });
+ };
+ agent._executeToolWithLifecycle = async (_tab, tool, args, _update, context) => {
+ if (tool === 'get_accessibility_tree') return { success: true, pageContent: pages.get(tab).tree ?? TREE };
+ if (tool === 'scroll') {
+ frames.main({ kind: 'dom', source: 'unknown', target: 'div#before-scroll' });
+ await beforePageAgentDispatch(api, tab, { kind: 'scroll', fenceOnly: true });
+ context._contentActionDispatchState.started = true;
+ }
+ dispatched.push({ tool, args });
+ return tool === 'done' ? { done: true, success: true, summary: args.summary, outcome: args.outcome }
+ : { success: true, dispatched: true, moved: true, movedWindow: true, scrollY: 600 };
+ };
+ try {
+ const options = { detachedRequestId: 'scroll-churn-' + tab, askStreamingEnabled: false };
+ const update = (type, data) => updates.push({ type, data });
+ const result = streaming ? await agent.processMessageStream(tab, 'Scroll down', update, 'act', options)
+ : await agent.processMessage(tab, 'Scroll down', update, 'act', [], options);
+ assert.equal(result, 'Scrolled down');
+ assert.equal(requests, 2);
+ assert.deepEqual(dispatched.map(item => item.tool), ['scroll', 'done']);
+ assert.deepEqual(dispatched[0].args, scenario.input, 'Keep the original pane/coordinate intent');
+ const prepared = pages.get(tab).preparations.filter(item => item.tool === 'scroll');
+ assert.equal(prepared.length, 1, 'There is one scroll preparation and dispatch');
+ assert.equal(prepared[0].allowPassiveRebase, true, 'The content dispatch fence can rebase automatic changes');
+ if (scenario.batch) {
+ const result = updates.find(event => event.type === 'tool_result' && event.data.name === 'browser_batch');
+ assert.equal(result?.data.result.success, true);
+ assert.equal(result.data.result.summary, '[browser_batch] ran 1 of 1 actions');
+ }
+ assert.notEqual(terminal, 'page_unstable');
+ assert.equal(agent._pageFeedbackRuns.size, 0);
+ } finally { pages.delete(tab); }
+ });
+ }
+ }
+
+ for (const streaming of [false, true]) {
+ for (const intervention of ['user_scroll', 'main_document_replacement']) {
+ test(`${build}: ${streaming ? 'stream' : 'chat'} ${intervention} still supersedes stale scroll intent`, async () => {
+ const tab = nextTab++, dispatched = [];
+ let agent, frames, requests = 0;
+ pages.set(tab, {});
+ agent = makeAgent(Agent, async () => {
+ requests++;
+ if (requests === 1) {
+ if (intervention === 'user_scroll') frames.main({ kind: 'scroll', source: 'user', target: 'html', viewport: { x: 0, y: 600 } });
+ else {
+ Object.assign(pages.get(tab), { documentId: 'main-document-next', documentToken: 'main-token-next' });
+ agent.observePageNavigation({ tabId: tab, frameId: 0, parentFrameId: -1,
+ documentId: 'main-document-next', url: PAGE_URL, transitionType: 'reload', transitionQualifiers: [] }, 'committed');
+ // The replacement content script joins the run before the
+ // refreshed observation and next model decision.
+ frames = bindFrames(agent, tab);
+ }
+ return call('stale-scroll', 'scroll', { direction: 'down', amount: 111 });
+ }
+ if (requests === 2) return call('fresh-scroll', 'scroll', { direction: 'down', amount: 222 });
+ assert.equal(requests, 3);
+ return call('finished', 'done', { summary: 'Scrolled from the fresh page', outcome: 'success' });
+ });
+ agent.maxSteps = 7;
+ agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => { frames = bindFrames(agent, tab); return { role: 'user', content }; };
+ agent._executeToolWithLifecycle = async (_tab, tool, args) => {
+ if (tool === 'get_accessibility_tree') return { success: true, pageContent: TREE };
+ dispatched.push({ tool, args });
+ return tool === 'done' ? { done: true, success: true, summary: args.summary, outcome: args.outcome }
+ : { success: true, dispatched: true, moved: true, movedWindow: true, scrollY: 222 };
+ };
+ try {
+ const options = { detachedRequestId: 'scroll-intervention-' + tab, askStreamingEnabled: false };
+ const result = streaming ? await agent.processMessageStream(tab, 'Scroll down', () => {}, 'act', options)
+ : await agent.processMessage(tab, 'Scroll down', () => {}, 'act', [], options);
+ assert.equal(result, 'Scrolled from the fresh page');
+ assert.equal(requests, 3, 'The model must reconsider scrolling after intervention or document replacement');
+ assert.deepEqual(dispatched.map(item => item.tool), ['scroll', 'done']);
+ assert.equal(dispatched[0].args.amount, 222, 'The pre-intervention scroll is never dispatched');
+ } finally { pages.delete(tab); }
+ });
+ }
+ }
+
+ test(`${build}: a deterministic scroll without model state dispatches on a quiet page`, async () => {
+ const tab = nextTab++, agent = makeAgent(Agent, async () => assert.fail('No inference'));
+ const dispatched = [];
+ const input = { direction: 'down', amount: 600 };
+ pages.set(tab, {});
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ bindFrames(agent, tab);
+ const run = agent._pageFeedbackRuns.get(tab);
+ assert.equal(run.modelState, undefined, 'Saved workflow replay has no pre-inference observation');
+ assert.equal(run.events.size, 0);
+ agent._executeToolWithLifecycle = async (_tab, tool, args) => {
+ assert.equal(tool, 'scroll');
+ await beforePageAgentDispatch(api, tab, { kind: 'scroll', fenceOnly: true });
+ dispatched.push({ tool, args });
+ return { success: true, dispatched: true, moved: true, movedWindow: true };
+ };
+ const result = await agent.executeTool(tab, 'scroll', input);
+ assert.equal(result.success, true);
+ assert.equal(result.dispatched, true);
+ assert.notEqual(result.pageFeedbackPending, true);
+ assert.deepEqual(dispatched, [{ tool: 'scroll', args: input }]);
+ assert.equal(run.modelState, undefined);
+ } finally { pages.delete(tab); agent._releaseRunEntry(tab); }
+ });
+
+ for (const change of ['steering_revision', 'run_mode']) {
+ for (const phase of ['preparation', 'dispatch_boundary']) {
+ test(`${build}: ${change} during scroll ${phase} fences dispatch even without page events`, async () => {
+ const tab = nextTab++, agent = makeAgent(Agent, async () => assert.fail('No inference'));
+ let handlerCalls = 0;
+ pages.set(tab, {});
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ agent._runModeOverrides.set(tab, 'act');
+ const steering = { acceptedIds: new Set(), messages: [] };
+ agent._steeringRuns.set(tab, steering);
+ bindFrames(agent, tab);
+ const messages = [{ role: 'system', content: '' }, { role: 'user', content: 'Scroll down' }];
+ agent.conversations.set(tab, messages);
+ await agent._capturePageFeedbackModelState(tab, messages);
+ const run = agent._pageFeedbackRuns.get(tab);
+ const changeAuthority = () => {
+ assert.equal(run.events.size, 0, 'The regression requires an empty page-feedback queue');
+ if (change === 'steering_revision') steering.acceptedIds.add('accepted-correction');
+ else agent._runModeOverrides.set(tab, 'ask');
+ };
+ pages.get(tab).onPrepare = params => {
+ if (params.tool === 'scroll' && phase === 'preparation') changeAuthority();
+ };
+ agent._executeToolWithLifecycle = async (_tab, tool) => {
+ if (tool === 'get_accessibility_tree') return { success: true, pageContent: TREE };
+ assert.equal(tool, 'scroll');
+ handlerCalls++;
+ assert.equal(phase, 'dispatch_boundary', 'A change during preparation rejects before the scroll handler');
+ changeAuthority();
+ await assert.rejects(beforePageAgentDispatch(api, tab, { kind: 'scroll', fenceOnly: true }),
+ error => error?.code === 'page_feedback_pending', 'The owner fence checks authority before its empty-queue return');
+ return { success: false, noDispatch: true, dispatched: false, pageFeedbackPending: true };
+ };
+ const result = await agent.executeTool(tab, 'scroll', { direction: 'down', amount: 600 });
+ assert.equal(result.noDispatch, true);
+ assert.equal(result.pageFeedbackPending, true);
+ assert.equal(handlerCalls, phase === 'preparation' ? 0 : 1);
+ assert.equal(run.events.size, 0);
+ assert.equal(pages.get(tab).preparations.filter(item => item.tool === 'scroll').length, 1);
+ } finally {
+ pages.delete(tab);
+ agent._steeringRuns.delete(tab);
+ agent._runModeOverrides.delete(tab);
+ agent._releaseRunEntry(tab);
+ }
+ });
+ }
+ }
+
+ if (build === 'chrome') {
+ test('chrome: the initial scroll preparation uses the same last-interaction pane as dispatch', async () => {
+ const tab = nextTab++, agent = makeAgent(Agent, async () => assert.fail('No inference'));
+ const dispatched = [];
+ pages.set(tab, {});
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ bindFrames(agent, tab);
+ const messages = [{ role: 'system', content: '' }, { role: 'user', content: 'Scroll down in this pane' }];
+ agent.conversations.set(tab, messages);
+ await agent._capturePageFeedbackModelState(tab, messages);
+ agent._lastInteractionRect.set(tab, { x: 300, y: 400, w: 40, h: 80, ts: Date.now(), url: PAGE_URL });
+ pages.get(tab).onPrepare = params => {
+ if (params.tool === 'scroll') assert.deepEqual({ x: params.x, y: params.y }, { x: 320, y: 440 },
+ 'The first content-monitor preparation must already target the last-interaction pane');
+ };
+ agent._executeToolWithLifecycle = async (_tab, tool, args) => {
+ if (tool === 'get_accessibility_tree') return { success: true, pageContent: TREE };
+ assert.equal(tool, 'scroll');
+ dispatched.push(args);
+ return { success: true, dispatched: true, moved: true, movedContainer: true };
+ };
+ assert.equal((await agent.executeTool(tab, 'scroll', { direction: 'down', amount: 600 })).success, true);
+ assert.deepEqual(dispatched, [{ direction: 'down', amount: 600, x: 320, y: 440, origin: 'last_interaction' }]);
+ const prepared = pages.get(tab).preparations.filter(item => item.tool === 'scroll');
+ assert.equal(prepared.length, 1);
+ assert.equal(prepared[0].allowPassiveRebase, true);
+ } finally { pages.delete(tab); agent._releaseRunEntry(tab); }
+ });
+ }
+
for (const [name, emit] of Object.entries(AUTOMATIC)) {
test(`${build}: a bound action revalidates through ${name} and dispatches on the first decision`, async () => {
const tab = nextTab++, updates = [], dispatched = [];
@@ -217,7 +478,8 @@ for (const build of ['chrome', 'firefox']) {
const update = (type, data) => updates.push({ type, data });
const result = streaming ? await agent.processMessageStream(tab, 'Add the blue shoes', update, 'act', options)
: await agent.processMessage(tab, 'Add the blue shoes', update, 'act', [], options);
- assert.match(result, /page kept changing/);
+ assert.match(result, /could not validate the next action/);
+ assert.doesNotMatch(result, /page kept changing/, 'A recovery message must describe failed validation rather than claim visible change');
assert.equal(terminal, 'page_unstable');
assert.equal(requests, 5);
assert.deepEqual(dispatched, [], 'A changed bound target is never dispatched');
diff --git a/test/feedback-navigation-idle.mjs b/test/feedback-navigation-idle.mjs
index c8981eff7..e73aca82d 100644
--- a/test/feedback-navigation-idle.mjs
+++ b/test/feedback-navigation-idle.mjs
@@ -220,7 +220,7 @@ for (const build of ['chrome', 'firefox']) {
const result = streaming ? await agent.processMessageStream(tab, 'Read the inbox headings', () => {}, 'act', options)
: await agent.processMessage(tab, 'Read the inbox headings', () => {}, 'act', [], options);
assert.doesNotMatch(result, /Stopped by user/, 'The idle gate starved the run until Stop');
- assert.match(result, /Read the inbox|page kept changing/);
+ assert.match(result, /Read the inbox|could not validate the next action after repeated browser updates/i);
assert.ok(Date.now() - started < 3000);
assert.equal(dispatched.filter(name => name === 'extract_data').length, 1, 'The read proceeds through automatic churn');
} finally { clearTimeout(stopper); clearInterval(ticker); urls.delete(tab); }
diff --git a/test/feedback-run-boundaries.mjs b/test/feedback-run-boundaries.mjs
index 34930e886..c5b9a7298 100644
--- a/test/feedback-run-boundaries.mjs
+++ b/test/feedback-run-boundaries.mjs
@@ -89,7 +89,7 @@ for (const build of ['chrome', 'firefox']) {
agent._enrichUserMessageWithCurrentPage = async (_tab, _messages, content) => { send = bindMain(agent, tab); return { role: 'user', content }; };
agent.executeTool = async (_tab, name) => name === 'get_accessibility_tree' ? { success: true, pageContent: 'button "Buy" [ref_5]' } : { success: true };
const first = await agent.processMessage(tab, 'Click the buy button for the blue shoes', () => {}, 'act', [], { detachedRequestId: 'first-' + tab });
- assert.match(first, /page kept changing/);
+ assert.match(first, /could not validate the next action after repeated browser updates/i);
assert.ok(requests.slice(1).every(messages => feedbackMessages(messages).length > 0), 'Within its run, feedback reaches each decision');
assert.deepEqual(feedbackMessages(agent.conversations.get(tab)), [], 'Run end removes its feedback from the conversation');
secondRun = true;
diff --git a/test/monitor-scroll-churn.mjs b/test/monitor-scroll-churn.mjs
new file mode 100644
index 000000000..838207a67
--- /dev/null
+++ b/test/monitor-scroll-churn.mjs
@@ -0,0 +1,239 @@
+// Relative scrolling must work on live pages without waiting for DOM quiet.
+// Exercise the real monitor and content scroll handler in both browser builds.
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import { test } from 'node:test';
+import { chromium, firefox } from 'playwright';
+
+const RUN = 'scroll-churn';
+const html = `<!doctype html><style>
+body{margin:0;font:16px sans-serif}#pane{position:absolute;left:20px;top:70px;width:700px;height:350px;overflow-y:auto}
+#other{position:absolute;left:750px;top:70px;width:200px;height:350px;overflow-y:auto}
+.row{height:45px}#counter{position:absolute;left:20px;top:20px}#human{position:absolute;left:750px;top:20px}
+</style><div id="counter">0</div><button id="human">Human control</button><section id="pane" role="region"></section><aside id="other"></aside>
+<script>for(const pane of document.querySelectorAll('#pane,#other'))pane.innerHTML=Array.from({length:50},(_,i)=>'<div class="row">Live row '+i+'</div>').join('');document.querySelector('#pane .row').id='origin';</script>`;
+
+async function fixture(browser, build) {
+ const context = await browser.newContext({ viewport: { width: 1000, height: 700 } });
+ await context.route('https://scroll.test/**', route => route.fulfill({ contentType: 'text/html', body: html }));
+ await context.addInitScript(RUN => {
+ window.feedback = []; window.messageListeners = [];
+ const runtime = {
+ onMessage: { addListener: fn => messageListeners.push(fn), removeListener() {} },
+ async sendMessage(message) {
+ if (message.action === 'get_page_monitor_state') return { active: true, runToken: RUN, documentToken: message.documentToken };
+ if (message.action === 'page_feedback') {
+ feedback.push(message.feedback);
+ // Keep outstanding feedback while the page continues rendering.
+ await new Promise(resolve => setTimeout(resolve, 300));
+ return { accepted: true };
+ }
+ return {};
+ },
+ };
+ window.chrome = { runtime }; window.browser = window.chrome;
+ window.__wb_ax_lookup = ref => document.getElementById(ref.replace(/^ref_/, ''));
+ window.deliver = (action, params = {}) => new Promise(resolve => {
+ for (const listener of messageListeners) {
+ let answered = false;
+ const asynchronous = listener({ target: 'content', action, params: { runToken: RUN, ...params } }, {}, value => { answered = true; resolve(value); });
+ if (answered || asynchronous === true) return;
+ }
+ resolve(undefined);
+ });
+ }, RUN);
+ for (const file of ['page-monitor.js', 'content.js']) await context.addInitScript({
+ content: fs.readFileSync(new URL(`../src/${build}/src/content/${file}`, import.meta.url), 'utf8'),
+ });
+ const page = await context.newPage();
+ await page.goto('https://scroll.test/start');
+ await page.waitForFunction(() => window.__wbPageMonitor?.active);
+ await page.waitForTimeout(500);
+ return { context, page };
+}
+
+const message = (page, action, params = {}) => page.evaluate(({ action, params }) => deliver(action, params), { action, params });
+const prepare = (page, extra = {}) => message(page, 'page_monitor_prepare', {
+ operationId: 'scroll', tool: 'scroll', allowPassiveRebase: true, ...extra,
+});
+const fence = page => message(page, 'page_monitor_dispatch', { operationId: 'scroll', kind: 'scroll', fenceOnly: true });
+
+for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
+ test(`${build}: scroll proceeds through live renders and preserves its document and pane fences`, { timeout: 120000 }, async t => {
+ const browser = await engine.launch({ headless: true });
+ try {
+ await t.test('unknown DOM and pending feedback do not stop the actual content scroll', async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ await prepare(page, { ref_id: 'ref_pane' });
+ await page.evaluate(() => {
+ __wbPageMonitor.dispatch({ operationId: 'unrelated', kind: 'dom', runToken: 'scroll-churn' });
+ document.getElementById('counter').textContent = '1';
+ window.churn = setInterval(() => document.getElementById('counter').textContent = String(Math.random()), 15);
+ });
+ assert.equal((await fence(page)).ready, true);
+ const result = await message(page, 'scroll', { ref_id: 'ref_pane', direction: 'down', amount: 120 });
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(result.movedContainer, true);
+ assert.equal(await page.locator('#pane').evaluate(node => node.scrollTop), 120);
+ assert.equal(await page.locator('#other').evaluate(node => node.scrollTop), 0);
+ const observed = await page.evaluate(() => { clearInterval(churn); return feedback; });
+ assert.ok(observed.some(event => event.kind === 'dom' && event.source === 'unknown'));
+ } finally { await context.close(); }
+ });
+
+ await t.test('recycling the row under coordinates preserves the same pane', async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ await prepare(page, { x: 100, y: 100 });
+ await page.locator('#pane .row').first().evaluate(node => { const replacement = node.cloneNode(true); replacement.textContent = 'Replacement live row'; node.replaceWith(replacement); });
+ assert.equal((await fence(page)).ready, true);
+ const result = await message(page, 'scroll', { x: 100, y: 100, direction: 'down', amount: 120 });
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(result.movedContainer, true);
+ } finally { await context.close(); }
+ });
+
+ for (const [name, point] of [
+ ['outside the ref pane', { x: 0, y: 0 }],
+ ['over another pane', { x: 800, y: 100 }],
+ ]) await t.test(`a ref takes precedence over fallback coordinates ${name}`, async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ const request = { ref_id: 'ref_pane', ...point, direction: 'down', amount: 120 };
+ assert.equal((await prepare(page, request)).ready, true);
+ // Replacing the unrelated pane must not change the ref's recipient,
+ // even when the ignored coordinates happen to hit that pane.
+ await page.locator('#other').evaluate(node => node.replaceWith(node.cloneNode(true)));
+ assert.equal((await fence(page)).ready, true);
+ const result = await message(page, 'scroll', request);
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(result.movedContainer, true);
+ assert.equal(await page.locator('#pane').evaluate(node => node.scrollTop), 120);
+ assert.equal(await page.locator('#other').evaluate(node => node.scrollTop), 0);
+ } finally { await context.close(); }
+ });
+
+ for (const direction of ['down', 'up']) await t.test(`a point outside all panes can scroll the largest visible container ${direction}`, async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ assert.equal(await page.evaluate(() => document.elementFromPoint(0, 0) === document.documentElement), true);
+ assert.equal(await page.evaluate(() => document.scrollingElement.scrollHeight <= window.innerHeight), true);
+ const before = await page.locator('#pane').evaluate((node, direction) => {
+ if (direction === 'up') node.scrollTop = node.scrollHeight;
+ return node.scrollTop;
+ }, direction);
+ const request = { x: 0, y: 0, direction, amount: 120 };
+ assert.equal((await prepare(page, request)).ready, true);
+ assert.equal((await fence(page)).ready, true);
+ const result = await message(page, 'scroll', request);
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(result.movedContainer, true);
+ assert.equal(await page.locator('#pane').evaluate(node => node.scrollTop), before + (direction === 'up' ? -120 : 120));
+ assert.equal(await page.locator('#other').evaluate(node => node.scrollTop), 0);
+ assert.equal(await page.evaluate(() => window.scrollY), 0);
+ } finally { await context.close(); }
+ });
+
+ await t.test('a nested pane at its limit can scroll its original ancestor', async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ await page.evaluate(() => {
+ const inner = document.createElement('div'); inner.id = 'inner';
+ inner.style.cssText = 'height:100px;overflow-y:auto'; inner.innerHTML = '<div style="height:600px">Nested history</div>';
+ document.getElementById('pane').prepend(inner); inner.scrollTop = inner.scrollHeight;
+ });
+ await prepare(page, { x: 100, y: 100 });
+ assert.equal((await fence(page)).ready, true);
+ const result = await message(page, 'scroll', { x: 100, y: 100, direction: 'down', amount: 120 });
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(await page.locator('#pane').evaluate(node => node.scrollTop), 120);
+ } finally { await context.close(); }
+ });
+
+ await t.test('generic window scrolling and explicit alsoWindow survive live page content', async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ await page.evaluate(() => { document.body.style.height = '2000px'; });
+ await prepare(page);
+ assert.equal((await fence(page)).ready, true);
+ const windowResult = await message(page, 'scroll', { direction: 'down', amount: 120 });
+ assert.equal(windowResult.success, true, JSON.stringify(windowResult));
+ assert.equal(windowResult.movedWindow, true);
+ await page.evaluate(() => { window.scrollTo(0, 0); document.getElementById('counter').textContent = 'Updated'; });
+ await prepare(page, { ref_id: 'ref_pane', alsoWindow: true });
+ assert.equal((await fence(page)).ready, true);
+ const combined = await message(page, 'scroll', { ref_id: 'ref_pane', direction: 'down', amount: 120, alsoWindow: true });
+ assert.equal(combined.success, true, JSON.stringify(combined));
+ assert.equal(combined.movedContainer, true);
+ assert.equal(combined.movedWindow, true);
+ } finally { await context.close(); }
+ });
+
+ for (const [name, request, mutate] of [
+ ['a replaced ref pane', { ref_id: 'ref_pane' }, page => page.locator('#pane').evaluate(node => node.replaceWith(node.cloneNode(true)))],
+ ['a ref moved into another pane', { ref_id: 'ref_origin' }, page => page.locator('#origin').evaluate(node => document.getElementById('other').prepend(node))],
+ ['a different pane under prepared coordinates', { x: 100, y: 100 }, page => page.locator('#pane').evaluate(node => { node.style.left = '750px'; document.getElementById('other').style.left = '20px'; })],
+ ['a replaced largest-container fallback pane', { x: 0, y: 0, direction: 'down' }, page => page.locator('#pane').evaluate(node => node.replaceWith(node.cloneNode(true)))],
+ ['a moved largest-container fallback pane', { x: 0, y: 0, direction: 'down' }, page => page.locator('#pane').evaluate(node => { node.style.top = '100px'; })],
+ ['a changed top-level viewport', {}, page => page.setViewportSize({ width: 990, height: 690 })],
+ ['a changed same-document URL', {}, page => page.evaluate(() => history.pushState({}, '', '/other'))],
+ ['a changed base URL', {}, page => page.evaluate(() => { const base = document.createElement('base'); base.href = '/other/'; document.head.append(base); })],
+ ['genuine user input', {}, page => page.click('#human')],
+ ]) await t.test(`${name} still vetoes the prepared scroll`, async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ await prepare(page, request);
+ await mutate(page);
+ const result = await fence(page);
+ assert.equal(result.ready, false, JSON.stringify(result));
+ assert.equal(result.pageFeedbackPending, true);
+ assert.equal(await page.locator('#pane').evaluate(node => node.scrollTop), 0);
+ } finally { await context.close(); }
+ });
+
+ await t.test('a second local prepare cannot erase the original human-intervention fence', async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ await prepare(page, { ref_id: 'ref_pane' });
+ assert.equal((await fence(page)).ready, true);
+ await page.click('#human');
+ await page.waitForTimeout(500);
+ const result = await message(page, 'scroll', { ref_id: 'ref_pane', direction: 'down', amount: 120 });
+ assert.notEqual(result.success, true, JSON.stringify(result));
+ assert.equal(await page.locator('#pane').evaluate(node => node.scrollTop), 0);
+ } finally { await context.close(); }
+ });
+
+ for (const [name, mutate] of [
+ ['replaced', page => page.locator('#pane').evaluate(node => node.replaceWith(node.cloneNode(true)))],
+ ['moved', page => page.locator('#pane').evaluate(node => { node.style.top = '100px'; })],
+ ]) await t.test(`a second local prepare cannot rebind a ${name} fallback pane`, async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ const request = { x: 0, y: 0, direction: 'down', amount: 120 };
+ assert.equal((await prepare(page, request)).ready, true);
+ assert.equal((await fence(page)).ready, true);
+ await mutate(page);
+ const result = await message(page, 'scroll', request);
+ assert.notEqual(result.success, true, JSON.stringify(result));
+ assert.equal(result.noDispatch, true, JSON.stringify(result));
+ assert.equal(await page.locator('#pane').evaluate(node => node.scrollTop), 0);
+ assert.equal(await page.locator('#other').evaluate(node => node.scrollTop), 0);
+ } finally { await context.close(); }
+ });
+
+ await t.test('scroll authority cannot authorize a click or input dispatch', async () => {
+ const { context, page } = await fixture(browser, build);
+ try {
+ await prepare(page, { ref_id: 'ref_pane' });
+ for (const kind of ['click', 'input']) {
+ const result = await message(page, 'page_monitor_dispatch', { operationId: 'scroll', kind, fenceOnly: true });
+ assert.equal(result.ready, false, JSON.stringify(result));
+ assert.equal(result.preparationVeto?.predicate, 'operation_kind_changed');
+ }
+ } finally { await context.close(); }
+ });
+ } finally { await browser.close(); }
+ });
+}
diff --git a/test/page-feedback-progress.mjs b/test/page-feedback-progress.mjs
index 5c6502a4f..b4791711c 100644
--- a/test/page-feedback-progress.mjs
+++ b/test/page-feedback-progress.mjs
@@ -272,12 +272,12 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(requests, 5); assert.equal(validations, 5); assert.equal(dispatched, 0);
if (condition === 'unsupported') {
assert.match(result, /could not confirm which part of the page the control belongs to/);
- assert.doesNotMatch(result, /page kept changing/);
+ assert.doesNotMatch(result, /could not validate the next action|page kept changing/i);
assert.equal(terminalStatus, 'action_context_unverified');
assert.ok(updates.some(u => u.type === 'warning' && /could not confirm/.test(u.data.message)));
assert.ok(modelMessages[1].some(m => /ACTION CONTEXT UNVERIFIED/.test(JSON.stringify(m))));
} else {
- assert.match(result, /page kept changing/);
+ assert.match(result, /could not validate the next action after repeated browser updates/i);
assert.equal(modelMessages.some(messages => messages.some(m => /ACTION CONTEXT UNVERIFIED/.test(JSON.stringify(m)))), false);
assert.equal(captures.some(c => c.actionTarget), false);
}
@@ -287,7 +287,7 @@ for (const build of ['chrome', 'firefox']) {
const retryMode = condition === 'unsupported' ? 'action_monitoring_relaxed' : 'resume';
assert.equal(offer.retryMode, retryMode);
assert.equal(offer.message, result);
- assert.match(result, /choose Retry/);
+ assert.match(result, /choose Retry/i);
assert.equal(agent._pendingFailureRecoveryForUi(tab).recoveryId, offer.recoveryId);
assert.deepEqual(agent.requestFailureRecovery(tab, offer.recoveryId, streaming ? 'retry' : 'give_up', 'owned-recovery-' + tab),
{ ok: true, continue: streaming, retryMode });
@@ -410,7 +410,7 @@ for (const build of ['chrome', 'firefox']) {
}
assert.deepEqual(dispatched, response === 'done' ? ['research_url', 'done'] : ['research_url']);
assert.ok(reads >= requests, 'Every pending passive batch is still observed');
- assert.equal(updates.some(update => update.type === 'warning' && /page kept changing/.test(update.data.message)), false);
+ assert.equal(updates.some(update => update.type === 'warning' && /could not validate the next action|page kept changing/i.test(update.data.message)), false);
assert.equal(agent._pageFeedbackRuns.size, 0);
});
}
@@ -549,7 +549,7 @@ for (const build of ['chrome', 'firefox']) {
assert.fail(`Unsafe ${name} must not reach dispatch`);
};
const result = await finish(agent, tab, streaming, updates, { detachedRequestId: 'progress-run', askStreamingEnabled: false });
- assert.match(result, /Stopped because the page kept changing/);
+ assert.match(result, /could not validate the next action after repeated browser updates/i);
assert.match(result, /task is incomplete/i);
assert.equal(requests, 5);
assert.equal(dispatched, 0);
@@ -558,7 +558,7 @@ for (const build of ['chrome', 'firefox']) {
assert.ok(messages.filter(message => message.webbrainAppOwnedKind === 'page_feedback').length <= 1,
'Counter churn must replace old observations instead of growing model context');
}
- assert.ok(updates.some(update => update.type === 'warning' && /Stopped because the page kept changing/.test(update.data.message)));
+ assert.ok(updates.some(update => update.type === 'warning' && /could not validate the next action after repeated browser updates/i.test(update.data.message)));
});
}
diff --git a/test/page-feedback-recovery.mjs b/test/page-feedback-recovery.mjs
index 7fe83b773..f3267760a 100644
--- a/test/page-feedback-recovery.mjs
+++ b/test/page-feedback-recovery.mjs
@@ -16,13 +16,14 @@ for (const browser of ['chrome', 'firefox']) {
assert.equal(attempts.at(-1).code, 'action_context_unverified');
assert.match(attempts.at(-1).message, /could not confirm which part of the page the control belongs to/);
assert.match(attempts.at(-1).message, /Open or focus the intended area, then choose Retry/);
- assert.doesNotMatch(attempts.at(-1).message, /page kept changing/);
+ assert.doesNotMatch(attempts.at(-1).message, /could not validate the next action|page kept changing/i);
assert.deepEqual(attempts.at(-1).metadata, { stage: 'response', streak: PAGE_FEEDBACK_STOP_AT,
toolNames: ['click_ax'], reason: 'target_context_unverified' });
const intervention = accountFeedbackSupersession(run, { passive: false, reason: 'target_context_unverified' });
assert.equal(intervention.stop, false); assert.equal(intervention.streak, 0);
const drift = Array.from({ length: PAGE_FEEDBACK_STOP_AT }, () => accountFeedbackSupersession(run, { passive: true })).at(-1);
- assert.equal(drift.code, PAGE_FEEDBACK_STOP_CODE); assert.match(drift.message, /page kept changing/);
+ assert.equal(drift.code, PAGE_FEEDBACK_STOP_CODE);
+ assert.match(drift.message, /could not validate the next action after repeated browser updates/i);
assert.equal(feedbackSupersessionMetadata({ reason: 'https://private.invalid/body' }).reason, undefined);
});
@@ -39,6 +40,8 @@ for (const browser of ['chrome', 'firefox']) {
assert.equal(attempts.at(-1).code, PAGE_FEEDBACK_STOP_CODE);
assert.match(attempts.at(-1).message, /task is incomplete/i);
assert.match(attempts.at(-1).message, /no stale action was dispatched/i);
+ assert.match(attempts.at(-1).message, /Choose Retry to inspect the current page and continue\./);
+ assert.doesNotMatch(attempts.at(-1).message, /page kept changing|wait for the page to settle/i);
});
test(`${browser}: changing tools or moving the race into preparation cannot evade the budget`, () => {
diff --git a/test/pure-calculation-reply.mjs b/test/pure-calculation-reply.mjs
index aa4d3d3c6..e80217339 100644
--- a/test/pure-calculation-reply.mjs
+++ b/test/pure-calculation-reply.mjs
@@ -206,7 +206,7 @@ for (const build of ['chrome', 'firefox']) {
assert.ok(requestsSeen.every(request => request.toolNames.includes('done')));
assert.equal(updates.filter(e => e.type === 'steering_applied').length, 1);
assert.deepEqual(updates.filter(e => e.type === 'tool_call').map(e => e.data.name), ['calculate', 'done']);
- assert.equal(updates.some(e => e.type === 'warning' && /page kept changing/.test(e.data.message)), false);
+ assert.equal(updates.some(e => e.type === 'warning' && /could not validate the next action|page kept changing/i.test(e.data.message)), false);
assert.equal(updates.filter(e => e.type === 'tool_result' && e.data.name === 'done').at(-1)?.data.result.outcome, 'success');
} finally { release(); documents.delete(tab); }
});
@@ -256,7 +256,7 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(requests, 4); assert.equal(calculations, 1);
assert.deepEqual(seen, [{ nudged: false, calculated: false }, { nudged: true, calculated: false }, { nudged: true, calculated: true }]);
assert.deepEqual(updates.filter(e => e.type === 'tool_call').map(e => e.data.name), ['calculate', 'done']);
- assert.equal(updates.some(e => e.type === 'warning' && /page kept changing/.test(e.data.message)), false);
+ assert.equal(updates.some(e => e.type === 'warning' && /could not validate the next action|page kept changing/i.test(e.data.message)), false);
} finally { release(); documents.delete(tab); }
});
}
@@ -332,7 +332,7 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(calculations, 1); assert.equal(requests, 3);
assert.deepEqual(updates.filter(e => e.type === 'tool_call').map(e => e.data.name), ['calculate']);
assert.equal(updates.some(e => e.type === 'tool_result' && e.data.name === 'done'), false);
- assert.doesNotMatch(result, /page kept changing/);
+ assert.doesNotMatch(result, /could not validate the next action|page kept changing/i);
// The chat answer is plain language; the guard's diagnostic stays in traces.
assert.match(result, /couldn't verify the requested result, so I stopped\. Nothing was changed, submitted, or sent\./);
assert.doesNotMatch(result, /\[|\]|recovery attempt|task tools/);
diff --git a/test/quality/ask-completion-churn-mv3.mjs b/test/quality/ask-completion-churn-mv3.mjs
index 7cb326638..e87ebd654 100644
--- a/test/quality/ask-completion-churn-mv3.mjs
+++ b/test/quality/ask-completion-churn-mv3.mjs
@@ -278,7 +278,7 @@ try {
assert.equal(current.running, false);
if (entry.dependent) {
assert.notEqual(delivered.run.result?.content, expected, 'Live-page answer cannot borrow conversation-only retention.');
- assert.match(delivered.run.result?.content || '', /page kept changing|page keeps changing|incomplete/i);
+ assert.match(delivered.run.result?.content || '', /could not validate the next action after repeated browser updates/i);
assert.equal(current.runUi?.runSucceeded, false);
assert.ok(main.length >= 1 && main.length <= 5); assert.ok(scope.length >= 1 && scope.length <= 5);
assert.equal(decisions.some(decision => decision.reason === 'page_independent_answer'), false);