Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11081: steer scout-act to observe, not evaluate-click; couple the gate to author-steer (#6623)
Andrew Neilsondde70127089b
diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.pyindex 40bb455d0..946889295 100644--- a/skyvern/forge/sdk/copilot/tools/guardrails.py+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py@@ -141,10 +141,11 @@ def _update_and_run_blocks_composition_evidence_precheck(_DOWNLOAD_SCOUT_REQUIRED_STEERING = (- "This block downloads a file, and a download fires only when the affordance is clicked. "- "Scout-act it first: call skyvern_evaluate to click the download control on the live page, then "- "re-author the block. The terminal download step will be compiled for you from the observed "- "target, so you do not author the expect_download idiom yourself."+ "This block downloads a file, and the terminal download step is compiled from the observed "+ "download affordance. Scout it first: reach the page that exposes the download control and "+ "call skyvern_evaluate to observe it (skyvern_evaluate cannot click — use the click tool to "+ "act), then re-author the block. The terminal download step will be compiled for you from the "+ "observed target, so you do not author the expect_download idiom yourself.")# After this many scout-act rejections in a turn, stop blocking and let the model halt honestly: a# download whose affordance the scout never resolves is not authorable, so looping author->scout@@ -174,6 +175,10 @@ def _download_scout_required_error(copilot_ctx: Any, workflow_yaml: str | None)synthesizer can compile the terminal download step. Off unless the scout-act flag is set."""if not settings.COPILOT_DOWNLOAD_SCOUT_ACT_REQUIRED_ENABLED:return None+ if not settings.COPILOT_REACHED_DOWNLOAD_TARGET_AUTHOR_STEER_ENABLED:+ # The gate is cleared by the reached-download scout interaction, which is only registered+ # when author-steer is on; enforcing it without that flag deadlocks until the retry cap.+ return Noneif copilot_ctx is None or workflow_yaml is None:return Nonedownload_labels = _download_intent_block_labels(workflow_yaml)diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex 05371c766..60f81ccdf 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -644,15 +644,17 @@ _EVALUATE_BASE_DESCRIPTION = ("Execute JavaScript in the browser and return the result. ""Use this to inspect DOM state, read values, or run arbitrary JS.")-# Scout-ACT framing: an interaction-gated affordance (a download, a row-expand, a post-login-# area) only reveals its result after it is clicked, so it must be acted on here rather than-# inspected passively. For a download the copilot then compiles the terminal download step.+# Scout-ACT framing: a download (or row-expand / post-login) affordance exposes its terminal+# target only once its page is reached. The model reaches that page with navigate/click and+# observes it here — evaluate cannot click — so the copilot can derive the target and compile+# the terminal download step._EVALUATE_SCOUT_ACT_DESCRIPTION = (_EVALUATE_BASE_DESCRIPTION- + " Some results only appear AFTER an interaction (a download, a row-expand, a post-login "- "area); scout-ACT those affordances here (click the control via JS) instead of inspecting "- "passively. For a download, click the download control with this tool rather than authoring "- "the download yourself — the copilot then compiles the terminal download step for you."+ + " Some affordances (a download, a row-expand, a post-login area) only expose their target "+ "once the page holding them is reached. Use this tool to OBSERVE that page — it cannot click; "+ "reach the page with the navigate/click tools first. For a download, observe the page that "+ "exposes the download control rather than authoring the download yourself; the copilot then "+ "compiles the terminal download step for you.")