Skip to content

Automate browser based workflows with AI

SKY-10929 Compile scouted Copilot actions at persist seam (#6534)

Andrew Neilson49cfcd96c2e0
diff --git a/skyvern-frontend/src/util/env.test.ts b/skyvern-frontend/src/util/env.test.ts
index 004fc1ac4..0d3d9dc44 100644
--- a/skyvern-frontend/src/util/env.test.ts
+++ b/skyvern-frontend/src/util/env.test.ts
@@ -1,3 +1,6 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+
import { afterEach, describe, expect, it, vi } from "vitest";
async function loadEnv(apiKey: string | null = null, streamingMode = "") {
@@ -44,6 +47,53 @@ describe("getCredentialParam", () => {
});
});
+describe("getRuntimeApiKey", () => {
+ // Built via join so this test file never contains the sentinel verbatim
+ // either — entrypoint-skyvernui.sh sed-replaces every occurrence in built
+ // assets, and the source guard below must stay meaningful.
+ const dockerSentinel = ["__SKYVERN_API_KEY", "_PLACEHOLDER__"].join("");
+
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ vi.resetModules();
+ window.sessionStorage.clear();
+ });
+
+ it("returns the build-time API key", async () => {
+ const { getRuntimeApiKey } = await loadEnv("eyJhbGciOiJIUzI1NiJ9.real.key");
+
+ expect(getRuntimeApiKey()).toBe("eyJhbGciOiJIUzI1NiJ9.real.key");
+ });
+
+ it("treats the .env.example placeholder as missing", async () => {
+ const { getRuntimeApiKey } = await loadEnv("YOUR_API_KEY");
+
+ expect(getRuntimeApiKey()).toBe(null);
+ });
+
+ it("treats an un-replaced docker placeholder as missing", async () => {
+ const { getRuntimeApiKey } = await loadEnv(dockerSentinel);
+
+ expect(getRuntimeApiKey()).toBe(null);
+ });
+
+ // Regression guard for the v1.0.34–v1.0.40 docker auth outage:
+ // entrypoint-skyvernui.sh sed-replaces EVERY occurrence of the docker
+ // sentinel in the built bundle with the real API key. When the full
+ // sentinel appeared verbatim in this module (inside the placeholder
+ // deny-list), the real key replaced it there too, so the key became a
+ // member of its own deny-list and the UI sent no x-api-key header at all
+ // (403 "Invalid credentials" on every request).
+ it("never spells out the full docker placeholder sentinel in module source", async () => {
+ const source = await readFile(
+ path.join(process.cwd(), "src/util/env.ts"),
+ "utf-8",
+ );
+
+ expect(source.includes(dockerSentinel)).toBe(false);
+ });
+});
+
describe("browserStreamingMode", () => {
afterEach(() => {
vi.unstubAllEnvs();
diff --git a/skyvern-frontend/src/util/env.ts b/skyvern-frontend/src/util/env.ts
index 1e85996e3..a76d2c5a6 100644
--- a/skyvern-frontend/src/util/env.ts
+++ b/skyvern-frontend/src/util/env.ts
@@ -80,12 +80,13 @@ function getRuntimeApiKey(): string | null {
// Treat placeholder / example values as missing so they are never sent
// as real credentials (e.g. from .env.example or an un-replaced Docker placeholder).
- const PLACEHOLDER_VALUES = [
- "YOUR_API_KEY",
- "__SKYVERN_API_KEY_PLACEHOLDER__",
- ];
- runtimeApiKey =
- candidate && PLACEHOLDER_VALUES.includes(candidate) ? null : candidate;
+ // The Docker sentinel is matched by its suffix on purpose: the prebuilt-image
+ // entrypoint sed-replaces every occurrence of the full sentinel in the bundle
+ // with the real API key, so spelling it out here would put the real key in its
+ // own deny-list and strip the x-api-key header from every request.
+ const isPlaceholder =
+ candidate === "YOUR_API_KEY" || candidate?.endsWith("_PLACEHOLDER__");
+ runtimeApiKey = candidate && isPlaceholder ? null : candidate;
return runtimeApiKey;
}
diff --git a/skyvern/config.py b/skyvern/config.py
index 651da2104..a15c2076e 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -148,6 +148,10 @@ class Settings(BaseSettings):
# Gate copilot verified-success on per-criterion outcome evidence.
# Off restores the prior run-status/suspicious-success gate and classifier prompt.
COPILOT_OUTCOME_VERIFICATION_ENABLED: bool = True
+ # Capture bounded page evidence synchronously in scout interaction post-hooks.
+ # Off restores schema-less interaction packets plus standalone page inspects.
+ COPILOT_SCOUT_ACT_OBSERVE_ENABLED: bool = True
+ COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS: float = 4.0
# Dispatch flag for the workflow copilot v2 (openai-agents-SDK rewrite).
# Off = existing direct-LLM copilot at workflow_copilot_chat_post.
# On = new agent-SDK path under skyvern.forge.sdk.copilot.
@@ -156,6 +160,9 @@ class Settings(BaseSettings):
# Experimental Workflow Copilot v2 branch mode.
# Off = standard block authoring. On = prefer code blocks for browser work.
WORKFLOW_COPILOT_CODE_BLOCK_MODE: bool = False
+ # Default-off companion to code-block mode. When enabled, Copilot can impose
+ # strict scout-trajectory synthesis at the persist seam for the narrow PR-1 envelope.
+ WORKFLOW_COPILOT_CODE_BLOCK_IMPOSE_SYNTHESIS: bool = False
PORT: int = 8000
ALLOWED_ORIGINS: list[str] = ["*"]
BLOCKED_HOSTS: list[str] = ["localhost"]
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 431e9a7d5..e4451fec0 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1203,7 +1203,10 @@ class AgentFunction:
def get_copilot_config(self, code_block_mode: bool | None = None) -> CopilotConfig | None:
"""Return an optional workflow copilot config override."""
resolved = settings.WORKFLOW_COPILOT_CODE_BLOCK_MODE if code_block_mode is None else code_block_mode
- return CopilotConfig(block_authoring_policy=block_authoring_policy_from_code_only_mode(resolved))
+ return CopilotConfig(
+ block_authoring_policy=block_authoring_policy_from_code_only_mode(resolved),
+ impose_synthesized_code_block=settings.WORKFLOW_COPILOT_CODE_BLOCK_IMPOSE_SYNTHESIS,
+ )
async def get_copilot_config_for_request(
self, organization_id: str | None = None, code_block_mode: bool | None = None
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 80a5f4370..4da9cd476 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -2904,6 +2904,7 @@ async def _run_copilot_turn_impl(
turn_index=turn_index,
prior_block_count=prior_block_count,
block_authoring_policy=copilot_config.block_authoring_policy,
+ impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,
)
# Fail loud if a future caller skips the kwarg and gets a fresh UUID from
# the default_factory — the envelope and terminal frames would then carry
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 192109a16..17d86b93f 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -14,10 +14,11 @@ import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from skyvern.forge.sdk.copilot.composition_evidence import SCOUT_INTERACTION_EVIDENCE_TOOL
-_MAX_STEPS = 20
+_MAX_STEPS = 60
_INDENT = " "
CREDENTIAL_FILL_TOOL_NAME = "fill_credential_field"
@@ -76,11 +77,20 @@ _POSITIONAL_RE = re.compile(
)
+@dataclass
+class SynthesisDiagnostics:
+ emitted_interaction_count: int = 0
+ truncated: bool = False
+ dropped_interactions: list[dict[str, Any]] = field(default_factory=list)
+ locator_provenance: list[dict[str, Any]] = field(default_factory=list)
+
+
@dataclass
class SynthesizedCodeBlock:
code: str
parameters: list[dict[str, str]] = field(default_factory=list)
notes: list[str] = field(default_factory=list)
+ diagnostics: SynthesisDiagnostics = field(default_factory=SynthesisDiagnostics)
# str.splitlines() and several parsers treat these codepoints as line boundaries, so a raw one in a
@@ -91,6 +101,44 @@ _EXTRA_LINE_SEPARATORS = ("\u2028", "\u2029")
_CONTROL_CODEPOINTS = frozenset(
chr(cp) for cp in (*range(0x00, 0x20), 0x7F, *range(0x80, 0xA0)) if chr(cp) not in ("\n", "\r")
)
+_SENSITIVE_URL_QUERY_RE = re.compile(
+ r"(?:password|passwd|token|secret|api[_-]?key|credential|auth|session|cookie)", re.I
+)
+
+
+def _scrub_url_for_code_literal(url: str) -> str:
+ try:
+ parts = urlsplit(url)
+ hostname = parts.hostname
+ port = parts.port
+ except ValueError:
+ return url
+ if not parts.scheme or not parts.netloc or hostname is None:
+ return url
+
+ netloc = hostname
+ if ":" in netloc and not netloc.startswith("["):
+ netloc = f"[{netloc}]"
+ if port is not None:
+ netloc = f"{netloc}:{port}"
+
+ query = urlencode(
+ [
+ (key, "__redacted__" if _SENSITIVE_URL_QUERY_RE.search(key) else value)
+ for key, value in parse_qsl(parts.query, keep_blank_values=True)
+ ],
+ doseq=True,
+ )
+ fragment = parts.fragment
+ fragment_pairs = parse_qsl(fragment, keep_blank_values=True) if "=" in fragment or "&" in fragment else []
+ if fragment_pairs:
+ fragment = urlencode(
+ [(key, "__redacted__" if _SENSITIVE_URL_QUERY_RE.search(key) else value) for key, value in fragment_pairs],
+ doseq=True,
+ )
+ elif fragment and _SENSITIVE_URL_QUERY_RE.search(fragment):
+ fragment = "__redacted__"
+ return urlunsplit((parts.scheme, netloc, parts.path, query, fragment))
def _py_str(value: str) -> str:
@@ -154,6 +202,11 @@ def _get_by_role_expr(role: str, name: str) -> str:
def _locator_expr(
interaction: Mapping[str, Any],
notes: list[str],
+ *,
+ diagnostics: SynthesisDiagnostics | None = None,
+ trajectory_index: int | None = None,
+ tool_name: str = "",
+ strict_selectors: bool = False,
) -> str:
"""Selector-first: emit the scout's captured working selector verbatim, and only fall back to an
ARIA get_by_role anchor when that selector is positional/index-based (and a role/name is available).
@@ -166,6 +219,29 @@ def _locator_expr(
role = str(interaction.get("role") or "").strip()
name = str(interaction.get("accessible_name") or "").strip()
+ if strict_selectors:
+ if not selector:
+ notes.append("dropped an interaction with no selector")
+ if diagnostics is not None:
+ diagnostics.dropped_interactions.append(
+ {
+ "trajectory_index": trajectory_index if trajectory_index is not None else -1,
+ "tool_name": tool_name,
+ "reason_code": "missing_selector",
+ }
+ )
+ return ""
+ if diagnostics is not None:
+ diagnostics.locator_provenance.append(
+ {
+ "trajectory_index": trajectory_index if trajectory_index is not None else -1,
+ "selector": selector,
+ "emitted_literal": selector,
+ "source": "selector",
+ }
+ )
+ return f"page.locator({_py_str(selector)})"
+
parsed = _parse_role_name(selector) if selector else None
if parsed is not None:
parsed_role, parsed_name, suffix = parsed
@@ -215,7 +291,9 @@ def _credential_param_key(interaction: Mapping[str, Any], used: set[str]) -> str
return _unique_key(_safe_param_base(name or "credential"), used)
-def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> SynthesizedCodeBlock | None:
+def synthesize_code_block(
+ trajectory: Sequence[Mapping[str, Any]], *, strict_selectors: bool = False
+) -> SynthesizedCodeBlock | None:
"""Deterministically synthesize a code block from a scout trajectory, or None if empty."""
if not trajectory:
return None
@@ -223,6 +301,7 @@ def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> Synthesize
lines: list[str] = []
notes: list[str] = []
parameters: list[dict[str, str]] = []
+ diagnostics = SynthesisDiagnostics()
used_param_keys: set[str] = set()
credential_param_keys: dict[str, str] = {}
@@ -237,12 +316,13 @@ def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> Synthesize
if entry_url:
if entry_index > 0:
notes.append("entry URL taken from a later interaction; earlier steps had no source_url")
- lines.append(f"{_INDENT}await page.goto({_py_str(entry_url)})")
+ lines.append(f"{_INDENT}await page.goto({_py_str(_scrub_url_for_code_literal(entry_url))})")
lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
emitted = 0
- for interaction in trajectory:
+ for trajectory_index, interaction in enumerate(trajectory):
if emitted >= _MAX_STEPS:
+ diagnostics.truncated = True
notes.append(f"trajectory truncated at {_MAX_STEPS} steps")
break
tool_name = str(interaction.get("tool_name") or "")
@@ -250,17 +330,47 @@ def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> Synthesize
if tool_name == "press_key":
key = str(interaction.get("key") or "").strip()
if not key:
+ diagnostics.dropped_interactions.append(
+ {"trajectory_index": trajectory_index, "tool_name": tool_name, "reason_code": "missing_key"}
+ )
continue
- locator = _locator_expr(interaction, notes) if interaction.get("selector") else ""
+ locator = (
+ _locator_expr(
+ interaction,
+ notes,
+ diagnostics=diagnostics,
+ trajectory_index=trajectory_index,
+ tool_name=tool_name,
+ strict_selectors=strict_selectors,
+ )
+ if interaction.get("selector")
+ else ""
+ )
if locator:
lines.append(f"{_INDENT}await {locator}.press({_py_str(key)})")
else:
+ if strict_selectors:
+ diagnostics.dropped_interactions.append(
+ {
+ "trajectory_index": trajectory_index,
+ "tool_name": tool_name,
+ "reason_code": "missing_selector",
+ }
+ )
+ continue
lines.append(f"{_INDENT}await page.keyboard.press({_py_str(key)})")
lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
emitted += 1
continue
- locator = _locator_expr(interaction, notes)
+ locator = _locator_expr(
+ interaction,
+ notes,
+ diagnostics=diagnostics,
+ trajectory_index=trajectory_index,
+ tool_name=tool_name,
+ strict_selectors=strict_selectors,
+ )
if not locator:
continue
@@ -276,6 +386,13 @@ def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> Synthesize
credential_field = str(interaction.get("credential_field") or "").strip()
if not credential_id or credential_field not in _CREDENTIAL_FIELDS:
notes.append("dropped a credential fill with no usable credential reference")
+ diagnostics.dropped_interactions.append(
+ {
+ "trajectory_index": trajectory_index,
+ "tool_name": tool_name,
+ "reason_code": "missing_credential_reference",
+ }
+ )
continue
credential_param_key = credential_param_keys.get(credential_id)
if credential_param_key is None:
@@ -287,19 +404,26 @@ def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> Synthesize
value = str(interaction.get("value") or "").strip()
if not value:
notes.append("dropped a select_option interaction with no recorded value")
+ diagnostics.dropped_interactions.append(
+ {"trajectory_index": trajectory_index, "tool_name": tool_name, "reason_code": "missing_value"}
+ )
continue
lines.append(f"{_INDENT}await {locator}.select_option({_py_str(value)})")
lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
else:
notes.append(f"skipped unsupported interaction tool_name={tool_name!r}")
+ diagnostics.dropped_interactions.append(
+ {"trajectory_index": trajectory_index, "tool_name": tool_name, "reason_code": "unsupported_tool"}
+ )
continue
emitted += 1
if not lines:
return None
+ diagnostics.emitted_interaction_count = emitted
code = "\n".join(lines) + "\n"
- return SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes)
+ return SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes, diagnostics=diagnostics)
# Model-owned slots the synthesizer cannot prove; the model fills these.
@@ -342,7 +466,7 @@ def build_artifact_metadata_skeleton(
"observation_refs": [observation_ref_id],
}
if entry_url_hint:
- page_dependency["url_hint"] = entry_url_hint
+ page_dependency["url_hint"] = _scrub_url_for_code_literal(entry_url_hint)
observation_ref: dict[str, Any] = {
"observation_ref": observation_ref_id,
diff --git a/skyvern/forge/sdk/copilot/config.py b/skyvern/forge/sdk/copilot/config.py
index 6e74fe3d4..f7e416d5f 100644
--- a/skyvern/forge/sdk/copilot/config.py
+++ b/skyvern/forge/sdk/copilot/config.py
@@ -379,6 +379,7 @@ class CopilotConfig:
enforcement_nudges: dict[str, str] = field(default_factory=_default_enforcement_nudges)
fallback_llm_key: str | None = field(default_factory=_default_fallback_llm_key)
block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARD
+ impose_synthesized_code_block: bool = False
def nudge(self, key: str) -> str:
return self.enforcement_nudges.get(key, DEFAULT_ENFORCEMENT_NUDGES[key])
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 7f3ac5321..12a50b2da 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -345,6 +345,7 @@ class CopilotContext(AgentContext):
allow_untested_workflow_draft: bool = False
request_policy: RequestPolicy | None = None
block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARD
+ impose_synthesized_code_block: bool = False
turn_intent: TurnIntent | None = None
turn_context_packet: TurnContextPacket | None = None
latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index cd80343e0..424b8aa14 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -203,6 +203,7 @@ class AgentContext:
allow_untested_workflow_draft: bool = False
request_policy: RequestPolicy | None = None
block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARD
+ impose_synthesized_code_block: bool = False
effective_workflow_proxy_location: Any | None = None
copilot_run_start_monotonic: float | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index 287bb8f27..b03aa544c 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import json
import time
from collections.abc import Mapping
from typing import Any
@@ -17,10 +18,17 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
COMPOSITION_STRIPPED_HTML_MAX_CHARS as _COMPOSITION_STRIPPED_HTML_MAX_CHARS,
)
-from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION as _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION,
+)
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS as _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema, parse_composition_structured
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.schemas.workflows import BlockType
@@ -649,3 +657,41 @@ async def _composition_get_html(copilot_ctx: Any, *, skip_raw: bool = False) ->
return stripped, None, truncated, True
error = html_result.get("error")
return "", str(error) if error else None, False, True
+
+
+async def _composition_get_structured_evidence(
+ copilot_ctx: Any,
+ *,
+ inspected_url: str,
+ current_url: str,
+ timeout_seconds: float = _DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+) -> dict[str, Any] | None:
+ """Capture composition evidence via the page-side extractor; None when it can't yield a usable payload."""
+ server = getattr(copilot_ctx, "discovery_mcp_server", None)
+ if server is None:
+ return None
+ with copilot_span("composition_structured_extract"):
+ try:
+ result = await asyncio.wait_for(
+ server.call_internal_tool(
+ "skyvern_evaluate", {"expression": _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION}
+ ),
+ timeout=timeout_seconds,
+ )
+ except Exception:
+ return None
+ if not isinstance(result, dict) or not result.get("ok"):
+ return None
+ raw = (result.get("data") or {}).get("result")
+ if isinstance(raw, str):
+ if len(raw) > _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS:
+ return None
+ try:
+ payload = json.loads(raw)
+ except (ValueError, TypeError):
+ return None
+ elif isinstance(raw, dict):
+ payload = raw
+ else:
+ return None
+ return parse_composition_structured(payload, inspected_url=inspected_url, current_url=current_url)
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index 4e8e50729..5153a7a98 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -15,12 +15,6 @@ from skyvern.forge.sdk.copilot.completion_verification import (
RunEvidenceSnapshot,
evaluate_completion_criteria,
)
-from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION as _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION,
-)
-from skyvern.forge.sdk.copilot.composition_browser_expressions import (
- COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS as _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS,
-)
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION as _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,
)
@@ -30,7 +24,6 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
merge_visual_composition_evidence,
page_evidence_needs_visual_fallback,
parse_composition_html,
- parse_composition_structured,
)
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY
@@ -44,6 +37,7 @@ from ._shared import (
_append_flow_evidence,
_composition_evidence_page_url,
_composition_get_html,
+ _composition_get_structured_evidence,
_discovery_extract_current_url,
_discovery_navigate,
_fallback_page_info,
@@ -605,43 +599,6 @@ async def _augment_composition_evidence_with_computed_obstruction_candidates(
return _merge_visual_obstruction_candidates(evidence, candidates)
-async def _composition_get_structured_evidence(
- copilot_ctx: Any,
- *,
- inspected_url: str,
- current_url: str,
-) -> dict[str, Any] | None:
- """Capture composition evidence via the page-side extractor; None when it can't yield a usable payload."""
- server = getattr(copilot_ctx, "discovery_mcp_server", None)
- if server is None:
- return None
- with copilot_span("composition_structured_extract"):
- try:
- result = await asyncio.wait_for(
- server.call_internal_tool(
- "skyvern_evaluate", {"expression": _COMPOSITION_STRUCTURED_EVIDENCE_EXPRESSION}
- ),
- timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
- )
- except Exception:
- return None
- if not isinstance(result, dict) or not result.get("ok"):
- return None
- raw = (result.get("data") or {}).get("result")
- if isinstance(raw, str):
- if len(raw) > _COMPOSITION_STRUCTURED_EVIDENCE_MAX_CHARS:
- return None
- try:
- payload = json.loads(raw)
- except (ValueError, TypeError):
- return None
- elif isinstance(raw, dict):
- payload = raw
- else:
- return None
- return parse_composition_structured(payload, inspected_url=inspected_url, current_url=current_url)
-
-
async def _capture_composition_evidence(
copilot_ctx: Any,
*,
diff --git a/skyvern/forge/sdk/copilot/tools/credential_fill.py b/skyvern/forge/sdk/copilot/tools/credential_fill.py
index aecc5427c..b4fa6a0a6 100644
--- a/skyvern/forge/sdk/copilot/tools/credential_fill.py
+++ b/skyvern/forge/sdk/copilot/tools/credential_fill.py
@@ -224,7 +224,7 @@ async def _fill_credential_field_impl(
credential_field=field,
credential_name=credential_name,
)
- observation_step = _register_scout_interaction_observation(
+ observation_step, _ = await _register_scout_interaction_observation(
copilot_ctx, tool_name="fill_credential_field", selector=selector, source_url=source_url, url=url
)
data: dict[str, Any] = {
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 546e93ccb..c0a2e2050 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -31,6 +31,7 @@ from .page_observation import (
_resolve_url_title,
)
from .scouting import (
+ _attach_scout_page_summary,
_capture_scout_source_url,
_clear_pending_browser_interaction_observation,
_consume_scout_source_url,
@@ -320,12 +321,14 @@ async def _click_post_hook(
role=role,
accessible_name=accessible_name,
)
- observation_step = _register_scout_interaction_observation(
+ observation_step, page_evidence = await _register_scout_interaction_observation(
ctx, tool_name="click", selector=data.get("selector", ""), source_url=source_url, url=url
)
if observation_step is not None:
result["observation_step"] = observation_step
result["data"]["observation_step"] = observation_step
+ if page_evidence is not None:
+ _attach_scout_page_summary(result, page_evidence)
return result
@@ -419,12 +422,14 @@ async def _type_text_post_hook(
role=role,
accessible_name=accessible_name,
)
- observation_step = _register_scout_interaction_observation(
+ observation_step, page_evidence = await _register_scout_interaction_observation(
ctx, tool_name="type_text", selector=selector, source_url=source_url, url=url
)
if observation_step is not None:
result["observation_step"] = observation_step
result["data"]["observation_step"] = observation_step
+ if page_evidence is not None:
+ _attach_scout_page_summary(result, page_evidence)
return result
@@ -511,12 +516,14 @@ async def _select_option_post_hook(
role=role,
accessible_name=accessible_name,
)
- observation_step = _register_scout_interaction_observation(
+ observation_step, page_evidence = await _register_scout_interaction_observation(
ctx, tool_name="select_option", selector=data.get("selector", ""), source_url=source_url, url=url
)
if observation_step is not None:
result["observation_step"] = observation_step
result["data"]["observation_step"] = observation_step
+ if page_evidence is not None:
+ _attach_scout_page_summary(result, page_evidence)
return result
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 4072d7d0d..a11c5fa91 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -1,12 +1,15 @@
from __future__ import annotations
import asyncio
+import json
import re
+import time
from typing import Any, cast
from urllib.parse import urlparse
import structlog
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
scout_accessible_role_name_expression as _scout_accessible_role_name_expression,
@@ -15,6 +18,7 @@ from skyvern.forge.sdk.copilot.composition_evidence import (
SCOUT_INTERACTION_EVIDENCE_TOOL,
has_bounded_page_schema,
)
+from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
PendingBrowserInteractionObservation,
@@ -24,6 +28,7 @@ from skyvern.forge.sdk.copilot.runtime import (
from ._shared import (
_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
_append_flow_evidence,
+ _composition_get_structured_evidence,
_same_page_ignoring_fragment,
_workflow_verification_evidence,
)
@@ -74,7 +79,7 @@ def _consume_pending_browser_interaction_observation(
return True
-_MAX_SCOUTED_INTERACTIONS = 20
+_MAX_SCOUTED_INTERACTIONS = 60
async def _live_working_page_url(ctx: AgentContext) -> str | None:
@@ -252,15 +257,51 @@ def _record_scouted_interaction(
)
-def _register_scout_interaction_observation(
+_ACT_OBSERVE_TOOLS = frozenset({"click"})
+
+
+async def _scout_act_observe_page_evidence(ctx: AgentContext, *, url: str) -> dict[str, Any] | None:
+ """Run the bounded page-side extractor right after a scout interaction.
+
+ Degrades to None on timeout, error, or a hollow parse so the interaction
+ result is never blocked or failed by capture problems."""
+ if getattr(ctx, "discovery_mcp_server", None) is None:
+ return None
+ timeout_seconds = settings.COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS
+ started = time.monotonic()
+ parsed: dict[str, Any] | None = None
+ try:
+ parsed = await _composition_get_structured_evidence(
+ ctx, inspected_url=url, current_url=url, timeout_seconds=timeout_seconds
+ )
+ if parsed is not None and has_bounded_page_schema(parsed):
+ outcome = "attached"
+ elif parsed is not None:
+ outcome = "hollow"
+ parsed = None
+ else:
+ outcome = "timeout" if time.monotonic() - started >= timeout_seconds else "error"
+ except Exception:
+ parsed = None
+ outcome = "error"
+ LOG.info(
+ "copilot_scout_act_observe",
+ outcome=outcome,
+ duration_ms=int((time.monotonic() - started) * 1000),
+ url=url,
+ )
+ return parsed
+
+
+async def _register_scout_interaction_observation(
ctx: AgentContext, *, tool_name: str, selector: str, source_url: str | None, url: str
-) -> int | None:
+) -> tuple[int | None, dict[str, Any] | None]:
# A successful scout interaction reaches the post-action page; record it as an
# interaction-reached observation so a click-reached block can be authored
# against it without a separate inspect_page_for_composition.
selector = selector.strip()
if not selector or not url:
- return None
+ return None, None
evidence: dict[str, Any] = {
"inspected_url": url,
"current_url": url,
@@ -270,7 +311,142 @@ def _register_scout_interaction_observation(
}
if source_url and source_url.strip():
evidence["interaction_source_url"] = source_url.strip()
- return _append_flow_evidence(ctx, evidence, reached_via="interaction")
+ page_evidence: dict[str, Any] | None = None
+ if settings.COPILOT_SCOUT_ACT_OBSERVE_ENABLED and tool_name in _ACT_OBSERVE_TOOLS:
+ parsed = await _scout_act_observe_page_evidence(ctx, url=url)
+ if parsed is not None:
+ # Identity keys overwrite the parsed packet so the entry stays a
+ # scout_interaction observation, with the schema merged before append.
+ evidence = {**parsed, **evidence}
+ page_evidence = evidence
+ # The schema is already attached; leaving the marker set would let a
+ # later evaluate/inspect mint a second interaction credit for one click.
+ _clear_pending_browser_interaction_observation(ctx)
+ step = _append_flow_evidence(ctx, evidence, reached_via="interaction")
+ return step, page_evidence
+
+
+_PAGE_SUMMARY_TEXT_CAP = 80
+_PAGE_SUMMARY_MAX_FIELDS = 8
+_PAGE_SUMMARY_MAX_SUBMITS = 4
+_PAGE_SUMMARY_MAX_NAV_TEXTS = 8
+_PAGE_SUMMARY_MAX_DISMISS_TEXTS = 4
+
+
+def _summary_text(value: Any) -> str:
+ return value.strip()[:_PAGE_SUMMARY_TEXT_CAP] if isinstance(value, str) else ""
+
+
+def _summary_field_name(field: dict[str, Any]) -> str:
+ for key in ("label", "name", "placeholder", "id"):
+ text = _summary_text(field.get(key))
+ if text:
+ return text
+ return ""
+
+
+def _build_scout_page_summary(evidence: dict[str, Any]) -> dict[str, Any]:
+ forms_summary: list[dict[str, Any]] = []
+ for form in evidence.get("forms") or []:
+ if not isinstance(form, dict):
+ continue
+ fields = [field for field in form.get("fields") or [] if isinstance(field, dict)]
+ submits = [control for control in form.get("submit_controls") or [] if isinstance(control, dict)]
+ forms_summary.append(
+ {
+ "field_count": len(fields),
+ "fields": [
+ name for name in (_summary_field_name(field) for field in fields[:_PAGE_SUMMARY_MAX_FIELDS]) if name
+ ],
+ "submit_controls": [
+ text
+ for text in (
+ _summary_text(control.get("text") or control.get("value"))
+ for control in submits[:_PAGE_SUMMARY_MAX_SUBMITS]
+ )
+ if text
+ ],
+ }
+ )
+ nav_targets = [target for target in evidence.get("navigation_targets") or [] if isinstance(target, dict)]
+ dismiss_texts: list[str] = []
+ for overlay in evidence.get("modal_overlays") or []:
+ if not isinstance(overlay, dict):
+ continue
+ for control in overlay.get("dismiss_controls") or []:
+ if len(dismiss_texts) >= _PAGE_SUMMARY_MAX_DISMISS_TEXTS:
+ break
+ if not isinstance(control, dict):
+ continue
+ text = _summary_text(control.get("text") or control.get("aria_label") or control.get("title"))
+ if text:
+ dismiss_texts.append(text)
+ challenge_state = evidence.get("challenge_state")
+ challenge_detected = bool(evidence.get("challenge_controls")) or (
+ isinstance(challenge_state, dict) and challenge_state.get("detected") is True
+ )
+ return {
+ "page_title": _summary_text(evidence.get("page_title")),
+ "forms": forms_summary,
+ "navigation_target_count": len(nav_targets),
+ "navigation_targets": [
+ text
+ for text in (_summary_text(target.get("text")) for target in nav_targets[:_PAGE_SUMMARY_MAX_NAV_TEXTS])
+ if text
+ ],
+ "result_container_count": len(evidence.get("result_containers") or []),
+ "challenge_detected": challenge_detected,
+ "modal_dismiss_controls": dismiss_texts,
+ }
+
+
+def _shed_scout_page_summary_section(summary: dict[str, Any]) -> bool:
+ """Drop one summary section, in fixed priority order; False when nothing is left to shed."""
+ if summary.get("navigation_targets"):
+ summary["navigation_targets"] = []
+ return True
+ forms = [form for form in summary.get("forms") or [] if isinstance(form, dict)]
+ for form in forms[1:]:
+ if form.get("fields"):
+ form["fields"] = []
+ return True
+ if summary.get("modal_dismiss_controls"):
+ summary["modal_dismiss_controls"] = []
+ return True
+ for form in forms[1:]:
+ if form.get("submit_controls"):
+ form["submit_controls"] = []
+ return True
+ if forms and forms[0].get("fields"):
+ fields = forms[0]["fields"]
+ forms[0]["fields"] = fields[: len(fields) // 2] if len(fields) > 2 else []
+ return True
+ if forms and forms[0].get("submit_controls"):
+ forms[0]["submit_controls"] = []
+ return True
+ if len(forms) > 1:
+ summary["forms"] = forms[:1]
+ return True
+ return False
+
+
+def _attach_scout_page_summary(result: dict[str, Any], page_evidence: dict[str, Any]) -> None:
+ """Attach a compact page summary at result["data"]["page"], keeping the whole
+ serialized result under the recent-output pruner cap by shedding sections —
+ never by slicing the serialized JSON."""
+ data = result.get("data")
+ if not isinstance(data, dict):
+ return
+ try:
+ summary = _build_scout_page_summary(page_evidence)
+ data["page"] = summary
+ while len(json.dumps(result)) > _RECENT_TOOL_OUTPUT_CHAR_CAP:
+ if not _shed_scout_page_summary_section(summary):
+ data.pop("page", None)
+ return
+ except Exception:
+ data.pop("page", None)
+ LOG.warning("copilot_scout_act_observe_summary_failed", exc_info=True)
def _mark_post_run_page_observed(ctx: AgentContext, *, source_tool: str, url: str) -> None:
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 31a89d7a8..13500bd1e 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -1,7 +1,11 @@
from __future__ import annotations
+import ast
import re
+import textwrap
from collections.abc import Mapping
+from dataclasses import dataclass
+from dataclasses import field as dataclass_field
from typing import Annotated, Any, Literal, NamedTuple
import structlog
@@ -10,7 +14,11 @@ from pydantic import AliasChoices, BaseModel, Field, ValidationError
from skyvern.forge import app
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
-from skyvern.forge.sdk.copilot.code_block_synthesis import artifact_dependency_id, artifact_observation_ref_id
+from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ artifact_dependency_id,
+ artifact_observation_ref_id,
+ synthesize_code_block,
+)
from skyvern.forge.sdk.copilot.composition_evidence import (
SCOUT_INTERACTION_EVIDENCE_TOOL,
composition_page_evidence_error,
@@ -648,6 +656,394 @@ def _code_seam_rejection_user_summary(*, metadata_rejected: bool, code_rejected:
return "I need to adjust how the workflow verifies its results before testing."
+@dataclass
+class _SynthesizedCodeImpositionResult:
+ workflow_yaml: str
+ substitutions: dict[str, Any] | None = None
+ violations: list[str] = dataclass_field(default_factory=list)
+
+
+_SUBMITTED_LITERAL_METHODS = frozenset({"fill", "type"})
+_SECRET_LIKE_LITERAL_RE = re.compile(
+ r"(?:password|passwd|token|secret|api[_-]?key|credential|bearer\s+|sk-[a-zA-Z0-9])",
+ re.I,
+)
+
+
+def _compiled_authoring_user_summary() -> str:
+ return "I need to bind the compiled browser-step code safely before saving this workflow."
+
+
+def _prior_yaml_source(ctx: AgentContext) -> tuple[str, str | None]:
+ last_yaml = getattr(ctx, "last_workflow_yaml", None)
+ if isinstance(last_yaml, str) and last_yaml:
+ return "last_workflow_yaml", last_yaml
+ workflow_yaml = getattr(ctx, "workflow_yaml", None)
+ return "workflow_yaml", workflow_yaml if isinstance(workflow_yaml, str) else None
+
+
+def _workflow_code_blocks(parsed: dict[str, Any]) -> list[dict[str, Any]]:
+ return [
+ block
+ for block in workflow_blocks(parsed)
+ if _enum_or_string_name(block.get("block_type")) == BlockType.CODE.value
+ ]
+
+
+def _submitted_code_block_changed(block: Mapping[str, Any], prior_yaml: str | None) -> bool:
+ label = str(block.get("label") or "").strip()
+ if not label or not prior_yaml:
+ return True
+ prior = parse_workflow_yaml(prior_yaml)
+ if not isinstance(prior, dict):
+ return True
+ for prior_block in _workflow_code_blocks(prior):
+ if str(prior_block.get("label") or "").strip() != label:
+ continue
+ return str(prior_block.get("code") or "") != str(block.get("code") or "")
+ return True
+
+
+def _wrapped_code_ast(code: str) -> ast.AST | None:
+ body = "\n".join(f" {line}" for line in code.splitlines())
+ if not body.strip():
+ body = " pass"
+ try:
+ return ast.parse(f"async def __submitted_code__():\n{body}\n")
+ except SyntaxError:
+ return None
+
+
+def _is_page_locator_expression(value: ast.AST) -> bool:
+ if not isinstance(value, ast.Call) or not isinstance(value.func, ast.Attribute):
+ return False
+ if value.func.attr not in {"locator", "get_by_role"}:
+ return False
+ return isinstance(value.func.value, ast.Name) and value.func.value.id == "page"
+
+
+def _single_assignment_string_literals(tree: ast.AST) -> dict[str, str]:
+ assignments: dict[str, list[str | None]] = {}
+
+ def record(name: str, value: ast.AST | None) -> None:
+ if isinstance(value, ast.Constant) and isinstance(value.value, str):
+ assignments.setdefault(name, []).append(value.value)
+ else:
+ assignments.setdefault(name, []).append(None)
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Assign):
+ for target in node.targets:
+ if isinstance(target, ast.Name):
+ record(target.id, node.value)
+ continue
+ if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
+ record(node.target.id, node.value)
+ continue
+ if isinstance(node, (ast.AugAssign, ast.NamedExpr)) and isinstance(node.target, ast.Name):
+ record(node.target.id, None)
+ continue
+ if isinstance(node, (ast.For, ast.AsyncFor)) and isinstance(node.target, ast.Name):
+ record(node.target.id, None)
+
+ return {name: values[0] for name, values in assignments.items() if len(values) == 1 and values[0] is not None}
+
+
+def _string_literal_argument(first_arg: ast.AST, assignment_literals: Mapping[str, str]) -> str | None:
+ if isinstance(first_arg, ast.Constant) and isinstance(first_arg.value, str):
+ return first_arg.value
+ if isinstance(first_arg, ast.Name):
+ return assignment_literals.get(first_arg.id)
+ if (
+ isinstance(first_arg, ast.Call)
+ and isinstance(first_arg.func, ast.Name)
+ and first_arg.func.id == "str"
+ and len(first_arg.args) == 1
+ and isinstance(first_arg.args[0], ast.Name)
+ and not first_arg.keywords
+ ):
+ return assignment_literals.get(first_arg.args[0].id)
+ return None
+
+
+def _submitted_fill_type_arguments(code: str) -> list[str | None]:
+ tree = _wrapped_code_ast(code)
+ if tree is None:
+ return []
+ assignment_literals = _single_assignment_string_literals(tree)
+ arguments: list[str | None] = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ func = node.func
+ if not isinstance(func, ast.Attribute) or func.attr not in _SUBMITTED_LITERAL_METHODS:
+ continue
+ if not _is_page_locator_expression(func.value):
+ arguments.append(None)
+ continue
+ if not node.args:
+ arguments.append(None)
+ continue
+ arguments.append(_string_literal_argument(node.args[0], assignment_literals))
+ return arguments
+
+
+def _safe_singleton_literal_for_parameter(
+ code: str, parameter_key: str, typed_length: int | None
+) -> tuple[str | None, str | None]:
+ arguments = _submitted_fill_type_arguments(code)
+ if len(arguments) != 1 or arguments[0] is None:
+ return (
+ None,
+ f"Unable to bind synthesized parameter `{parameter_key}`: submitted code must contain exactly one direct browser-locator string literal fill/type call or single local string constant binding.",
+ )
+ literal = arguments[0]
+ if _SECRET_LIKE_LITERAL_RE.search(literal):
+ return None, f"Unable to bind synthesized parameter `{parameter_key}`: submitted literal looks credential-like."
+ if typed_length is not None and typed_length > 0 and len(literal) != typed_length:
+ return (
+ None,
+ f"Unable to bind synthesized parameter `{parameter_key}`: submitted literal length does not match the scout record.",
+ )
+ return literal, None
+
+
+def _is_credential_parameter(parameter: Mapping[str, Any]) -> bool:
+ parameter_type = str(parameter.get("parameter_type") or "").lower()
+ workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
+ return parameter_type == "credential" or (
+ parameter_type == "workflow" and workflow_parameter_type == "credential_id"
+ )
+
+
+def _submitted_string_parameter_default(
+ parameters: list[Any],
+ *,
+ synthesized_key: str,
+ typed_length: int | None,
+) -> tuple[dict[str, Any] | None, str | None]:
+ candidates: list[dict[str, Any]] = []
+ for parameter in parameters:
+ if not isinstance(parameter, dict):
+ continue
+ candidate_key = str(parameter.get("key") or "").strip()
+ if not candidate_key or candidate_key == synthesized_key:
+ continue
+ if _is_credential_parameter(parameter):
+ continue
+ parameter_type = str(parameter.get("parameter_type") or "").lower()
+ workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
+ if parameter_type and parameter_type != "workflow":
+ continue
+ if workflow_parameter_type and workflow_parameter_type != "string":
+ continue
+ default_value = parameter.get("default_value")
+ if isinstance(default_value, str):
+ candidates.append(parameter)
+
+ if len(candidates) != 1:
+ return None, None
+
+ candidate = candidates[0]
+ default_value = str(candidate.get("default_value") or "")
+ if _SECRET_LIKE_LITERAL_RE.search(default_value):
+ return (
+ None,
+ f"Unable to bind synthesized parameter `{synthesized_key}`: submitted parameter default looks credential-like.",
+ )
+ if typed_length is not None and typed_length > 0 and len(default_value) != typed_length:
+ return (
+ None,
+ f"Unable to bind synthesized parameter `{synthesized_key}`: submitted parameter default length does not match the scout record.",
+ )
+ return candidate, None
+
+
+def _reconcile_synthesized_parameters(
+ *,
+ parsed: dict[str, Any],
+ code_block: dict[str, Any],
+ submitted_code: str,
+ synthesized_parameters: list[dict[str, str]],
+ scout_trajectory: list[ScoutedInteraction],
+) -> tuple[list[str], list[str]]:
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, dict):
+ return [], ["Unable to bind synthesized parameters: workflow_definition is missing."]
+ parameters = workflow_definition.get("parameters")
+ if parameters is None:
+ parameters = []
+ workflow_definition["parameters"] = parameters
+ if not isinstance(parameters, list):
+ return [], ["Unable to bind synthesized parameters: workflow_definition.parameters must be a list."]
+
+ existing_by_key = {
+ str(param.get("key")): param for param in parameters if isinstance(param, dict) and param.get("key")
+ }
+ existing_credentials = credential_params(parameters)
+ parameter_keys: list[str] = []
+ violations: list[str] = []
+ non_credential_synthesized = [param for param in synthesized_parameters if not param.get("credential_id")]
+ typed_lengths = [
+ int(interaction.get("typed_length") or 0)
+ for interaction in scout_trajectory
+ if str(interaction.get("tool_name") or "") == "type_text"
+ ]
+
+ for synthesized_param in synthesized_parameters:
+ key = str(synthesized_param.get("key") or "").strip()
+ if not key:
+ violations.append("Unable to bind synthesized parameter: parameter key is missing.")
+ continue
+ if key in parameter_keys:
+ violations.append(f"Unable to bind synthesized parameter `{key}`: duplicate synthesized key.")
+ continue
+ parameter_keys.append(key)
+
+ credential_id = str(synthesized_param.get("credential_id") or "").strip()
+ existing = existing_by_key.get(key)
+ if credential_id:
+ if existing is not None:
+ if existing_credentials.get(key) != credential_id:
+ violations.append(
+ f"Unable to bind synthesized credential parameter `{key}`: submitted credential binding does not match scout metadata."
+ )
+ continue
+ parameters.append(
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "credential_id",
+ "key": key,
+ "default_value": credential_id,
+ }
+ )
+ continue
+
+ if existing is not None:
+ if _is_credential_parameter(existing):
+ violations.append(
+ f"Unable to bind synthesized parameter `{key}`: submitted parameter is credential-typed."
+ )
+ continue
+
+ if len(non_credential_synthesized) != 1:
+ violations.append(
+ f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous."
+ )
+ continue
+
+ typed_length = typed_lengths[0] if len(typed_lengths) == 1 else None
+ submitted_parameter, error = _submitted_string_parameter_default(
+ parameters,
+ synthesized_key=key,
+ typed_length=typed_length,
+ )
+ if error:
+ violations.append(error)
+ continue
+ if submitted_parameter is not None:
+ submitted_parameter["parameter_type"] = "workflow"
+ submitted_parameter["workflow_parameter_type"] = "string"
+ submitted_parameter["key"] = key
+ continue
+
+ literal, error = _safe_singleton_literal_for_parameter(submitted_code, key, typed_length)
+ if error:
+ violations.append(error)
+ continue
+ parameters.append(
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": key,
+ "default_value": literal,
+ }
+ )
+
+ code_block["parameter_keys"] = parameter_keys
+ return parameter_keys, violations
+
+
+def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext) -> _SynthesizedCodeImpositionResult:
+ if not getattr(ctx, "impose_synthesized_code_block", False):
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
+ if getattr(ctx, "update_workflow_called", False):
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
+
+ scout_trajectory = getattr(ctx, "scout_trajectory", None)
+ if not isinstance(scout_trajectory, list) or not scout_trajectory:
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
+ first_source_url = str(scout_trajectory[0].get("source_url") or "").strip()
+ if not first_source_url:
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
+
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
+ code_blocks = _workflow_code_blocks(parsed)
+ if len(code_blocks) != 1:
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
+ code_block = code_blocks[0]
+ submitted_code = str(code_block.get("code") or "")
+
+ prior_source, prior_yaml = _prior_yaml_source(ctx)
+ if not _submitted_code_block_changed(code_block, prior_yaml):
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
+
+ synthesized = synthesize_code_block(scout_trajectory, strict_selectors=True)
+ if synthesized is None:
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=["Unable to impose synthesized code block: scout trajectory produced no runnable code."],
+ )
+
+ diagnostics = synthesized.diagnostics
+ violations: list[str] = []
+ if diagnostics.truncated:
+ violations.append("Unable to impose synthesized code block: scout trajectory was truncated.")
+ for dropped in diagnostics.dropped_interactions:
+ reason = str(dropped.get("reason_code") or "unknown")
+ tool_name = str(dropped.get("tool_name") or "unknown")
+ index = dropped.get("trajectory_index", "?")
+ violations.append(
+ f"Unable to impose synthesized code block: dropped scout interaction {index} from `{tool_name}` ({reason})."
+ )
+ for provenance in diagnostics.locator_provenance:
+ if provenance.get("source") != "selector" or provenance.get("selector") != provenance.get("emitted_literal"):
+ violations.append("Unable to impose synthesized code block: locator provenance was not byte-equal.")
+ break
+
+ parameter_keys, parameter_violations = _reconcile_synthesized_parameters(
+ parsed=parsed,
+ code_block=code_block,
+ submitted_code=submitted_code,
+ synthesized_parameters=synthesized.parameters,
+ scout_trajectory=scout_trajectory,
+ )
+ violations.extend(parameter_violations)
+ if violations:
+ return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml, violations=violations)
+
+ code_block["code"] = textwrap.dedent(synthesized.code).lstrip("\n")
+ credential_parameter_keys = [
+ str(param.get("key") or "") for param in synthesized.parameters if str(param.get("credential_id") or "").strip()
+ ]
+ substitutions = {
+ "block_label": str(code_block.get("label") or ""),
+ "source_trajectory_count": len(scout_trajectory),
+ "parameter_keys": parameter_keys,
+ "credential_parameter_keys": credential_parameter_keys,
+ "selector_provenance": diagnostics.locator_provenance,
+ "prior_source": prior_source,
+ }
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=yaml.safe_dump(parsed, sort_keys=False),
+ substitutions=substitutions,
+ )
+
+
def _code_artifact_metadata_shape_errors(label: str, artifact: Mapping[str, Any]) -> list[str]:
"""Return every shape violation for one artifact; the caller aggregates them."""
errors: list[str] = []
@@ -854,6 +1250,15 @@ async def _update_workflow(
ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))
ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))
ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))
+ imposition = _maybe_impose_synthesized_code_block(workflow_yaml, ctx)
+ if imposition.violations:
+ return {
+ "ok": False,
+ "error": "\n".join(imposition.violations),
+ "user_facing_summary": _compiled_authoring_user_summary(),
+ }
+ workflow_yaml = imposition.workflow_yaml
+ params["workflow_yaml"] = workflow_yaml
scout_trajectory = getattr(ctx, "scout_trajectory", None)
normalization = _normalize_code_artifact_metadata_detailed(
params.get("code_artifact_metadata"),
@@ -1039,12 +1444,15 @@ async def _update_workflow(
await emit_workflow_draft(ctx.stream, ctx, workflow)
except Exception as emit_err:
LOG.warning("copilot_narrative_workflow_draft_emit_failed", error=str(emit_err))
+ data: dict[str, Any] = {
+ "message": "Workflow updated successfully.",
+ "block_count": len(workflow.workflow_definition.blocks) if workflow.workflow_definition else 0,
+ }
+ if imposition.substitutions is not None:
+ data["imposed_substitutions"] = imposition.substitutions
return {
"ok": True,
- "data": {
- "message": "Workflow updated successfully.",
- "block_count": len(workflow.workflow_definition.blocks) if workflow.workflow_definition else 0,
- },
+ "data": data,
"_workflow": workflow,
}
except (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException) as e:
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 7254ef548..325b496ae 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -18,6 +18,8 @@ from skyvern.forge.sdk.copilot.tools import (
_detect_stale_block_metadata,
_update_workflow,
)
+from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
+from skyvern.forge.sdk.copilot.workflow_credential_utils import parse_workflow_yaml, workflow_blocks
def _yaml(body: str) -> str:
@@ -49,6 +51,74 @@ _SAFE_CODE_YAML = _yaml(
"""
)
+_SUBMITTED_LITERAL_YAML = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.locator("input[placeholder='Search']").fill("Taylor Brooks")
+ await page.locator("button.lookup").click()
+ """
+)
+
+_SUBMITTED_COMPUTED_LITERAL_YAML = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.locator("input[placeholder='Search']").fill(provider_name)
+ """
+)
+
+_SUBMITTED_LOCAL_CONSTANT_YAML = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ provider_query = "Taylor Brooks"
+ await page.locator("input[placeholder='Search']").fill(str(provider_query))
+ """
+)
+
+_SUBMITTED_COMPUTED_PARAMETER_YAML = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: provider_query
+ default_value: Taylor Brooks
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.locator("input[placeholder='Search']").fill(str(provider_query))
+ """
+)
+
+_SUBMITTED_MIXED_LITERAL_YAML = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.locator("input[placeholder='Search']").fill("Taylor Brooks")
+ await page.locator("#other").fill(provider_name)
+ """
+)
+
def _code_only_ctx() -> CopilotContext:
ctx = CopilotContext(
@@ -77,6 +147,31 @@ def _standard_ctx() -> CopilotContext:
return ctx
+def _enable_imposition(ctx: CopilotContext) -> None:
+ ctx.impose_synthesized_code_block = True
+
+
+def _stub_successful_update(monkeypatch: pytest.MonkeyPatch) -> None:
+ def _fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
+ return SimpleNamespace(
+ workflow_definition=SimpleNamespace(blocks=[SimpleNamespace(label="search_registry")]),
+ proxy_location=None,
+ )
+
+ async def _fake_get_prior_workflow(_ctx: CopilotContext) -> None:
+ return None
+
+ monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", _fake_process_workflow_yaml)
+ monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", _fake_get_prior_workflow)
+ monkeypatch.setattr(workflow_update_module, "composition_page_evidence_error", lambda *_args, **_kwargs: None)
+
+
+def _single_code_block(parsed: dict[str, object]) -> dict[str, object]:
+ blocks = [block for block in workflow_blocks(parsed) if str(block.get("block_type") or "").lower() == "code"]
+ assert len(blocks) == 1
+ return blocks[0]
+
+
def _credential_code_yaml(*, code: str, credential_id: str = "cred_missing") -> str:
indented_code = textwrap.indent(textwrap.dedent(code).strip(), " " * 8)
return (
@@ -170,6 +265,164 @@ class TestCodeSafetySeam:
assert ctx.code_artifact_metadata == {}
+class TestCompiledAuthoringImposition:
+ def _provider_search_ctx(self) -> CopilotContext:
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#provInput",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 13,
+ "role": "textbox",
+ "accessible_name": "Provider Name",
+ "trajectory_index": 0,
+ }
+ ]
+ return ctx
+
+ @pytest.mark.asyncio
+ async def test_imposes_strict_scout_selector_and_lifts_singleton_literal(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
+ assert "input[placeholder='Search']" not in block["code"]
+ assert block["parameter_keys"] == ["provider_name"]
+ parameters = parsed["workflow_definition"]["parameters"]
+ assert parameters == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ "default_value": "Taylor Brooks",
+ }
+ ]
+ assert result["data"]["imposed_substitutions"] == {
+ "block_label": "search_registry",
+ "source_trajectory_count": 1,
+ "parameter_keys": ["provider_name"],
+ "credential_parameter_keys": [],
+ "selector_provenance": [
+ {
+ "trajectory_index": 0,
+ "selector": "#provInput",
+ "emitted_literal": "#provInput",
+ "source": "selector",
+ }
+ ],
+ "prior_source": "workflow_yaml",
+ }
+
+ @pytest.mark.asyncio
+ async def test_unchanged_prior_code_does_not_impose(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ ctx.workflow_yaml = _SUBMITTED_LITERAL_YAML
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
+
+ assert result["ok"] is True
+ assert "imposed_substitutions" not in result["data"]
+ assert ctx.workflow_yaml == _SUBMITTED_LITERAL_YAML
+
+ @pytest.mark.asyncio
+ async def test_flag_off_code_only_mode_does_not_impose(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ ctx.impose_synthesized_code_block = False
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_LITERAL_YAML}, ctx)
+
+ assert result["ok"] is True
+ assert "imposed_substitutions" not in result["data"]
+ assert ctx.workflow_yaml == _SUBMITTED_LITERAL_YAML
+
+ @pytest.mark.asyncio
+ async def test_unbound_synthesized_parameter_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_COMPUTED_LITERAL_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `provider_name`" in result["error"]
+ assert ctx.workflow_yaml == ""
+
+ @pytest.mark.asyncio
+ async def test_single_local_string_constant_is_lifted_for_synthesized_key(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_LOCAL_CONSTANT_YAML}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
+ assert block["parameter_keys"] == ["provider_name"]
+ assert parsed["workflow_definition"]["parameters"] == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ "default_value": "Taylor Brooks",
+ }
+ ]
+
+ @pytest.mark.asyncio
+ async def test_single_submitted_string_parameter_is_adopted_for_synthesized_key(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ ctx.scout_trajectory[0]["accessible_name"] = "Search by doctor name or specialty, hospital, procedure, and more"
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_COMPUTED_PARAMETER_YAML}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ synthesized_key = "search_by_doctor_name_or_specialty_hospital_procedure_and_more"
+ assert f'await page.locator("#provInput").fill(str({synthesized_key}))' in block["code"]
+ assert block["parameter_keys"] == [synthesized_key]
+ assert parsed["workflow_definition"]["parameters"] == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": synthesized_key,
+ "default_value": "Taylor Brooks",
+ }
+ ]
+ assert result["data"]["imposed_substitutions"]["parameter_keys"] == [synthesized_key]
+
+ @pytest.mark.asyncio
+ async def test_mixed_literal_and_computed_fill_rejects_before_persist(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LITERAL_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert "exactly one direct browser-locator string literal fill/type call" in result["error"]
+ assert ctx.workflow_yaml == ""
+
+
class TestSeamSalvageIntoContext:
@pytest.mark.asyncio
async def test_conforming_label_persists_when_sibling_label_rejects(self) -> None:
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 907d2c5d1..2f956f2ce 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -12,6 +12,7 @@ from typing import Any
from skyvern.forge.sdk.copilot.code_block_preflight import preflight_code_block
from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ _MAX_STEPS,
_SYNTHESIZED_BLOCK_LABEL,
build_synthesized_artifact_metadata,
render_synthesized_offer_text,
@@ -368,16 +369,114 @@ class TestTrajectoryFidelity:
for key in keys:
assert f"fill(str({key}))" in result.code
- def test_step_cap_truncates_at_twenty(self) -> None:
+ def test_step_cap_truncates_at_configured_limit(self) -> None:
trajectory = [
_interaction("click", selector=f'role=button[name="b{i}"]', source_url="https://example.com/")
- for i in range(25)
+ for i in range(_MAX_STEPS + 5)
]
result = synthesize_code_block(trajectory)
assert result is not None
- assert result.code.count(".click()") == 20
+ assert result.code.count(".click()") == _MAX_STEPS
+ assert result.diagnostics.truncated is True
assert any("truncated" in note for note in result.notes)
+ def test_strict_synthesis_emits_byte_equal_selector_provenance(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "type_text",
+ selector="#provInput",
+ source_url="https://example.com/find-care",
+ typed_length=13,
+ role="textbox",
+ accessible_name="Provider Name",
+ )
+ ],
+ strict_selectors=True,
+ )
+
+ assert result is not None
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in result.code
+ assert result.diagnostics.dropped_interactions == []
+ assert result.diagnostics.locator_provenance == [
+ {
+ "trajectory_index": 0,
+ "selector": "#provInput",
+ "emitted_literal": "#provInput",
+ "source": "selector",
+ }
+ ]
+
+ def test_strict_synthesis_reports_unsupported_interaction(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction("click", selector="#open", source_url="https://example.com/"),
+ _interaction("hover", selector="#menu", source_url="https://example.com/"),
+ ],
+ strict_selectors=True,
+ )
+
+ assert result is not None
+ assert result.diagnostics.dropped_interactions == [
+ {"trajectory_index": 1, "tool_name": "hover", "reason_code": "unsupported_tool"}
+ ]
+
+ def test_synthesis_scrubs_credentials_from_emitted_url_literals(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector="#go",
+ source_url="https://user:[email protected]/search?token=secret-token&q=provider#access_token=fragment-token&section=results",
+ )
+ ]
+ )
+ metadata = build_synthesized_artifact_metadata(
+ [
+ _interaction(
+ "click",
+ selector="#go",
+ source_url="https://user:[email protected]/search?token=secret-token&q=provider#access_token=fragment-token&section=results",
+ )
+ ]
+ )
+
+ assert result is not None
+ assert "user:password" not in result.code
+ assert "secret-token" not in result.code
+ assert "fragment-token" not in result.code
+ assert "q=provider" in result.code
+ assert "section=results" in result.code
+ page_dependency = metadata["page_dependencies"][0]
+ assert page_dependency["url_hint"] == (
+ "https://example.com/search?token=__redacted__&q=provider#access_token=__redacted__&section=results"
+ )
+
+ def test_synthesis_scrubs_bare_sensitive_url_fragments(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector="#go",
+ source_url="https://example.com/search?q=provider#secret-token-fragment",
+ )
+ ]
+ )
+ metadata = build_synthesized_artifact_metadata(
+ [
+ _interaction(
+ "click",
+ selector="#go",
+ source_url="https://example.com/search?q=provider#secret-token-fragment",
+ )
+ ]
+ )
+
+ assert result is not None
+ assert "secret-token-fragment" not in result.code
+ assert 'await page.goto("https://example.com/search?q=provider#__redacted__")' in result.code
+ assert metadata["page_dependencies"][0]["url_hint"] == "https://example.com/search?q=provider#__redacted__"
+
class TestDeterminismAndEmpty:
def test_empty_trajectory_returns_none(self) -> None:
diff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.py
index 3f46da982..6accdbd2e 100644
--- a/tests/unit/test_copilot_fill_credential_field.py
+++ b/tests/unit/test_copilot_fill_credential_field.py
@@ -261,7 +261,11 @@ def _wire_impl(
monkeypatch.setattr(credential_fill_module, "_tool_loop_error", lambda *a, **k: None)
monkeypatch.setattr(credential_fill_module, "_authority_tool_error", lambda *a, **k: None)
monkeypatch.setattr(credential_fill_module, "record_tool_step_result_for_ctx", lambda *a, **k: None)
- monkeypatch.setattr(credential_fill_module, "_register_scout_interaction_observation", lambda *a, **k: 3)
+
+ async def fake_register(*_args: object, **_kwargs: object) -> tuple[int, None]:
+ return 3, None
+
+ monkeypatch.setattr(credential_fill_module, "_register_scout_interaction_observation", fake_register)
class TestFillCredentialFieldImpl:
diff --git a/tests/unit/test_copilot_scout_act_observe.py b/tests/unit/test_copilot_scout_act_observe.py
new file mode 100644
index 000000000..2eade1c79
--- /dev/null
+++ b/tests/unit/test_copilot_scout_act_observe.py
@@ -0,0 +1,379 @@
+"""Act-and-observe scouting (SKY-10932): a navigating scout click runs the bounded
+page-side extractor synchronously and merges the schema into the same
+scout_interaction flow-evidence packet before append, degrading to today's
+schema-less packet on timeout/error/hollow parses."""
+
+from __future__ import annotations
+
+import asyncio
+import copy
+import json
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.config import settings
+from skyvern.forge.sdk.copilot.composition_evidence import (
+ _auto_credit_interaction_observation,
+ has_bounded_page_schema,
+)
+from skyvern.forge.sdk.copilot.enforcement import _RECENT_TOOL_OUTPUT_CHAR_CAP
+from skyvern.forge.sdk.copilot.tools import _click_post_hook
+from skyvern.forge.sdk.copilot.tools.scouting import (
+ _consume_pending_browser_interaction_observation,
+)
+
+_SOURCE_URL = "https://example.com/product"
+_LANDING_URL = "https://example.com/results"
+
+_SCHEMA_LESS_PACKET_KEYS = {
+ "inspected_url",
+ "current_url",
+ "source_tool",
+ "interaction_tool",
+ "interaction_selector",
+ "interaction_source_url",
+}
+
+
+def _bounded_extractor_payload() -> dict[str, Any]:
+ return {
+ "page_title": "Results",
+ "forms": [
+ {
+ "fields": [
+ {"name": "npi", "label": "NPI number", "type": "text", "selector": "#npi"},
+ {"name": "state", "label": "State", "type": "select", "selector": "#state"},
+ ],
+ "submit_controls": [{"text": "Search", "type": "submit", "selector": "#go"}],
+ }
+ ],
+ "navigation_targets": [
+ {"text": "Provider details", "href": f"{_LANDING_URL}/details", "selector": "a.details"}
+ ],
+ "result_containers": [{"tag": "table", "id": "results", "selector": "#results"}],
+ "modal_overlays": [
+ {
+ "role": "dialog",
+ "selector": ".cookie-banner",
+ "dismiss_controls": [{"tag": "button", "text": "Accept", "selector": ".cookie-accept"}],
+ }
+ ],
+ }
+
+
+def _ctx(*, server: Any = None, source_url: str | None = _SOURCE_URL) -> SimpleNamespace:
+ return SimpleNamespace(
+ pending_browser_interaction_observation=None,
+ discovery_mcp_server=server,
+ scouted_interactions=[],
+ scout_trajectory=[],
+ pending_scout_source_url=source_url,
+ flow_evidence=[],
+ )
+
+
+def _server_returning(payload: dict[str, Any]) -> SimpleNamespace:
+ server = SimpleNamespace()
+ server.call_internal_tool = AsyncMock(return_value={"ok": True, "data": {"result": payload}})
+ return server
+
+
+async def _run_click(ctx: SimpleNamespace) -> dict[str, Any]:
+ return await _click_post_hook(
+ {"ok": True, "data": {"selector": "#open-details"}},
+ {"browser_context": {"url": _LANDING_URL, "title": "Results"}},
+ ctx,
+ )
+
+
+def _flow_by_step(ctx: SimpleNamespace) -> dict[int, tuple[dict[str, Any], str]]:
+ return {entry["step"]: (entry["evidence"], entry["reached_via"]) for entry in ctx.flow_evidence}
+
+
+class TestActObserveSuccess:
+ @pytest.mark.asyncio
+ async def test_schema_merged_into_interaction_packet_before_append(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ ctx = _ctx(server=_server_returning(_bounded_extractor_payload()))
+
+ result = await _run_click(ctx)
+
+ assert result["ok"] is True
+ assert len(ctx.flow_evidence) == 1
+ entry = ctx.flow_evidence[0]
+ assert entry["had_bounded_schema"] is True
+ assert entry["reached_via"] == "interaction"
+ evidence = entry["evidence"]
+ assert evidence["source_tool"] == "scout_interaction"
+ assert evidence["interaction_tool"] == "click"
+ assert evidence["interaction_selector"] == "#open-details"
+ assert evidence["interaction_source_url"] == _SOURCE_URL
+ assert evidence["inspected_url"] == _LANDING_URL
+ assert evidence["current_url"] == _LANDING_URL
+ assert has_bounded_page_schema(evidence)
+ assert evidence["forms"][0]["fields"][0]["label"] == "NPI number"
+
+ @pytest.mark.asyncio
+ async def test_pending_marker_cleared_and_result_carries_summary(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ ctx = _ctx(server=_server_returning(_bounded_extractor_payload()))
+
+ result = await _run_click(ctx)
+
+ assert ctx.pending_browser_interaction_observation is None
+ assert result["observation_step"] == ctx.flow_evidence[0]["step"]
+ assert result["data"]["observation_step"] == ctx.flow_evidence[0]["step"]
+ page = result["data"]["page"]
+ assert page["page_title"] == "Results"
+ assert page["forms"] == [
+ {
+ "field_count": 2,
+ "fields": ["NPI number", "State"],
+ "submit_controls": ["Search"],
+ }
+ ]
+ assert page["navigation_target_count"] == 1
+ assert page["navigation_targets"] == ["Provider details"]
+ assert page["result_container_count"] == 1
+ assert page["challenge_detected"] is False
+ assert page["modal_dismiss_controls"] == ["Accept"]
+ assert len(json.dumps(result)) <= _RECENT_TOOL_OUTPUT_CHAR_CAP
+
+
+class TestActObserveDegrade:
+ @pytest.mark.asyncio
+ async def test_timeout_degrades_to_schema_less_packet_and_keeps_marker(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS", 0.05)
+
+ async def slow_extract(*_args: object, **_kwargs: object) -> dict[str, Any]:
+ await asyncio.sleep(0.25)
+ return {"ok": True, "data": {"result": _bounded_extractor_payload()}}
+
+ server = SimpleNamespace()
+ server.call_internal_tool = AsyncMock(side_effect=slow_extract)
+ ctx = _ctx(server=server)
+
+ result = await _run_click(ctx)
+
+ assert result["ok"] is True
+ assert result["data"] == {
+ "selector": "#open-details",
+ "url": _LANDING_URL,
+ "title": "Results",
+ "observation_step": ctx.flow_evidence[0]["step"],
+ }
+ assert "page" not in result["data"]
+ entry = ctx.flow_evidence[0]
+ assert entry["had_bounded_schema"] is False
+ assert set(entry["evidence"].keys()) == _SCHEMA_LESS_PACKET_KEYS
+ assert ctx.pending_browser_interaction_observation is not None
+ assert ctx.pending_browser_interaction_observation.tool_name == "click"
+ assert ctx.pending_browser_interaction_observation.url == _LANDING_URL
+
+ @pytest.mark.asyncio
+ async def test_hollow_parse_degrades_to_schema_less_packet(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ ctx = _ctx(server=_server_returning({"page_title": "Loading", "forms": []}))
+
+ result = await _run_click(ctx)
+
+ assert result["ok"] is True
+ assert "page" not in result["data"]
+ assert ctx.flow_evidence[0]["had_bounded_schema"] is False
+ assert set(ctx.flow_evidence[0]["evidence"].keys()) == _SCHEMA_LESS_PACKET_KEYS
+ assert ctx.pending_browser_interaction_observation is not None
+
+ @pytest.mark.asyncio
+ async def test_extractor_error_never_fails_the_click(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ server = SimpleNamespace()
+ server.call_internal_tool = AsyncMock(side_effect=RuntimeError("browser gone"))
+ ctx = _ctx(server=server)
+
+ result = await _run_click(ctx)
+
+ assert result["ok"] is True
+ assert "page" not in result["data"]
+ assert ctx.flow_evidence[0]["had_bounded_schema"] is False
+
+
+class TestActObserveNoRace:
+ @pytest.mark.asyncio
+ async def test_appended_entry_never_mutates_after_hook_returns(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_TIMEOUT_SECONDS", 0.05)
+
+ async def slow_extract(*_args: object, **_kwargs: object) -> dict[str, Any]:
+ await asyncio.sleep(0.2)
+ return {"ok": True, "data": {"result": _bounded_extractor_payload()}}
+
+ server = SimpleNamespace()
+ server.call_internal_tool = AsyncMock(side_effect=slow_extract)
+ ctx = _ctx(server=server)
+
+ await _run_click(ctx)
+ snapshot = copy.deepcopy(ctx.flow_evidence[0])
+ assert snapshot["had_bounded_schema"] is False
+
+ await asyncio.sleep(0.4)
+
+ assert ctx.flow_evidence[0] == snapshot
+ assert ctx.flow_evidence[0]["had_bounded_schema"] is False
+ assert not has_bounded_page_schema(ctx.flow_evidence[0]["evidence"])
+
+ @pytest.mark.asyncio
+ async def test_successful_attach_credits_exactly_once(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ ctx = _ctx(server=_server_returning(_bounded_extractor_payload()))
+
+ await _run_click(ctx)
+
+ # The pending marker was consumed by the synchronous attach, so a later
+ # evaluate/inspect on the same page cannot mint a second interaction packet.
+ later_inspect_evidence = {
+ "current_url": _LANDING_URL,
+ "forms": [{"fields": [{"name": "npi"}]}],
+ }
+ assert (
+ _consume_pending_browser_interaction_observation(
+ ctx, current_url=_LANDING_URL, evidence=later_inspect_evidence
+ )
+ is False
+ )
+
+ by_step = _flow_by_step(ctx)
+ consumed: set[int] = set()
+ assert _auto_credit_interaction_observation(by_step, consumed) is True
+ credited_evidence, _ = by_step[next(iter(consumed))]
+ assert has_bounded_page_schema(credited_evidence)
+ assert _auto_credit_interaction_observation(by_step, consumed) is False
+
+ @pytest.mark.asyncio
+ async def test_degraded_path_preserves_pending_upgrade(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ ctx = _ctx(server=_server_returning({"page_title": "Loading", "forms": []}))
+
+ await _run_click(ctx)
+
+ later_inspect_evidence = {
+ "current_url": _LANDING_URL,
+ "forms": [{"fields": [{"name": "npi"}]}],
+ }
+ assert (
+ _consume_pending_browser_interaction_observation(
+ ctx, current_url=_LANDING_URL, evidence=later_inspect_evidence
+ )
+ is True
+ )
+ assert ctx.pending_browser_interaction_observation is None
+
+
+class TestActObserveSummaryBound:
+ @pytest.mark.asyncio
+ async def test_adversarial_page_never_clips_serialized_result(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+ payload = {
+ "page_title": "R" * 300,
+ "forms": [
+ {
+ "fields": [
+ {
+ "name": f"field_{form_index}_{field_index}",
+ "label": f"Label {form_index}-{field_index} " + "x" * 240,
+ "type": "text",
+ "selector": f"#f{form_index}-{field_index}",
+ }
+ for field_index in range(20)
+ ],
+ "submit_controls": [
+ {"text": f"Submit {form_index} " + "y" * 120, "type": "submit"} for _ in range(10)
+ ],
+ }
+ for form_index in range(5)
+ ],
+ "navigation_targets": [
+ {
+ "text": f"Nav {nav_index} " + "z" * 160,
+ "href": f"{_LANDING_URL}/nav/{nav_index}",
+ "selector": f"a.nav-{nav_index}",
+ }
+ for nav_index in range(20)
+ ],
+ "result_containers": [
+ {"tag": "table", "id": f"results-{index}", "selector": f"#results-{index}"} for index in range(8)
+ ],
+ "modal_overlays": [
+ {
+ "role": "dialog",
+ "selector": f".overlay-{index}",
+ "dismiss_controls": [
+ {"tag": "button", "text": f"Dismiss {index}-{control} " + "w" * 100} for control in range(6)
+ ],
+ }
+ for index in range(5)
+ ],
+ }
+ ctx = _ctx(server=_server_returning(payload))
+
+ result = await _run_click(ctx)
+
+ assert result["ok"] is True
+ serialized = json.dumps(result)
+ assert len(serialized) <= _RECENT_TOOL_OUTPUT_CHAR_CAP
+ assert json.loads(serialized) == result
+ # The flow-evidence packet keeps the full schema; only the tool result is compact.
+ assert ctx.flow_evidence[0]["had_bounded_schema"] is True
+ assert len(ctx.flow_evidence[0]["evidence"]["forms"]) == 5
+
+
+class TestActObserveKillSwitch:
+ @pytest.mark.asyncio
+ async def test_disabled_restores_two_call_behavior(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", False)
+ server = _server_returning(_bounded_extractor_payload())
+ ctx = _ctx(server=server)
+
+ result = await _run_click(ctx)
+
+ server.call_internal_tool.assert_not_awaited()
+ assert result["ok"] is True
+ assert result["data"] == {
+ "selector": "#open-details",
+ "url": _LANDING_URL,
+ "title": "Results",
+ "observation_step": ctx.flow_evidence[0]["step"],
+ }
+ entry = ctx.flow_evidence[0]
+ assert entry["had_bounded_schema"] is False
+ assert set(entry["evidence"].keys()) == _SCHEMA_LESS_PACKET_KEYS
+ assert ctx.pending_browser_interaction_observation is not None
+
+ @pytest.mark.asyncio
+ async def test_non_click_tools_do_not_capture(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.copilot import tools as tools_module
+
+ monkeypatch.setattr(settings, "COPILOT_SCOUT_ACT_OBSERVE_ENABLED", True)
+
+ async def passes(*_args: object, **_kwargs: object) -> None:
+ return None
+
+ monkeypatch.setattr(tools_module.mcp_hooks, "_verify_scout_type_landed", passes)
+ server = _server_returning(_bounded_extractor_payload())
+ ctx = _ctx(server=server, source_url=_SOURCE_URL)
+
+ result = await tools_module._type_text_post_hook(
+ {"ok": True, "data": {"selector": 'role=textbox[name="Search"]', "text_length": 8}},
+ {"browser_context": {"url": _LANDING_URL, "title": "Results"}},
+ ctx,
+ )
+
+ server.call_internal_tool.assert_not_awaited()
+ assert result["ok"] is True
+ assert "page" not in result["data"]
+ assert ctx.flow_evidence[0]["had_bounded_schema"] is False