Skip to content

Automate browser based workflows with AI

Remove the Task V3 humanized-input experiment, keeping one plain input module (#8769)

pedrohsdb29a04e9bf36a
diff --git a/skyvern/config.py b/skyvern/config.py
index 44af15dd5..47edff2f2 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -542,10 +542,6 @@ class Settings(BaseSettings):
# year-only segment groups to the segment path (SKY-17013). Force-on term only: runs are randomized per
# run by the flag of the same name, read through run_arm_enabled(DATE_SEGMENT_AIM_FLAG, ...).
TASK_V3_DATE_SEGMENT_AIM: bool = False
- # Send Task V3's clicks, typing and scrolls through EventStrategyFactory, as v1 does, so a run follows
- # whatever USE_EVENT_STRATEGIES registered. Force-on term only: runs are randomized per run by
- # the flag of the same name, read through run_arm_enabled(HUMANIZED_INPUT_FLAG, ...). Off: plain Playwright.
- TASK_V3_HUMANIZED_INPUT: bool = False
# Render the previous block's outcome (status / finish reason / final URL) and whether this is the
# last block into a v3 block's goal. Costs prompt tokens on every turn of the block, so it is
# measured via taskv3_block_context_tokens before it earns default-on. The outcome itself is
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index c2e4217b7..9e5a43462 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -185,7 +185,6 @@ from skyvern.forge.sdk.workflow.models.credential_release import CredentialRelea
from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
from skyvern.forge.sdk.workflow.page_derived_templates import NO_RENDER_RECORD, UNVERIFIED_ROOT_CLASSES
-from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3.goal_check import (
GOAL_CHECK_PROMPT_NAME,
PRE_JUDGE_SKIP_REASONS,
@@ -200,7 +199,6 @@ from skyvern.forge.taskv3.run_arms import (
EXTRACTION_REPORTS_FLAG,
GOAL_CHECK_ENFORCE_FLAG,
GOAL_CHECK_FLAG,
- HUMANIZED_INPUT_FLAG,
resolve_run_arm,
run_arm_enabled,
)
@@ -2326,15 +2324,6 @@ class ForgeAgent:
organization_id=task.organization_id,
forced=settings.TASK_V3_EXTRACTION_REPORTS,
)
- await resolve_run_arm(
- context,
- HUMANIZED_INPUT_FLAG,
- distinct_id=task.workflow_run_id or task.task_id,
- organization_id=task.organization_id,
- forced=settings.TASK_V3_HUMANIZED_INPUT,
- )
- # Here as well as on the loop's finish line, so a run whose loop raises still records its profile.
- LOG.info("Task V3 humanized input resolved", **input_dispatch.arm_fields())
# The judge's finish-time screenshot, reused as the decision screenshot of an accepted completion.
goal_judge_shot: list[bytes] = []
page_free_validation = bool(
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 9a36b6530..db85443b9 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -205,6 +205,43 @@ class BuildTestPacketRunBrowser(BaseModel):
note: str
+LoopSelectedInput = Literal["loop_variable_reference", "loop_over_parameter_key", "none"]
+
+
+class LoopInputFact(BaseModel):
+ """One for_loop's declared input in the executed definition, and the loop_values its run rows recorded."""
+
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ block_label: str
+ enclosing_loop_label: str | None = None
+ loop_over_parameter_key: str | None = None
+ producer_block_label: str | None = None
+ producer_output_parameter_id: str | None = None
+ loop_variable_reference: str | None = None
+ selected_input: LoopSelectedInput
+ run_rows: int | None = None
+ loop_values_counts: list[int | Literal["not_recorded"]] | None = None
+ loop_values_counts_omitted: int | None = None
+
+
+class LoopInputs(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ workflow_run_id: str | None = None
+ workflow_id: str
+ workflow_permanent_id: str
+ version: int
+ loops: list[LoopInputFact]
+ loops_omitted: int | None = None
+
+
+class LoopInputsUnavailable(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ executed_definition: Literal["unavailable"] = "unavailable"
+
+
class BuildTestPacketRun(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
@@ -214,6 +251,7 @@ class BuildTestPacketRun(BaseModel):
browser: BuildTestPacketRunBrowser | None = None
execution_source: dict[str, Any] | None = None
browser_start: dict[str, Any] | None = None
+ loop_inputs: LoopInputs | LoopInputsUnavailable | None = None
class BuildTestPacketPageState(BaseModel):
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 69a4d6951..cd27d0e30 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -35,6 +35,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestPacketPageState,
BuildTestPacketRegisteredOutput,
BuildTestPacketRequestedOutput,
+ LoopInputs,
append_omission_notice,
coerce_block_action_observations,
coerce_block_end_urls,
@@ -67,6 +68,7 @@ _INTERNAL_RUN_OUTCOME_RECORDED_KEY = "_copilot_internal_run_outcome_recorded"
_INTERNAL_GOAL_PATH_OMISSIONS_KEY = "_copilot_internal_goal_path_omissions"
_BASE64_IMAGE_OMITTED_MESSAGE = "[base64 image omitted — screenshot was taken successfully]"
BUILD_TEST_PACKET_KEY = "build_test_packet"
+LOOP_INPUTS_KEY = "loop_inputs"
_BUILD_TEST_PACKET_MAX_CHARS = 47_000
_BUILD_TEST_WORKFLOW_MAX_CHARS = 30_000
@@ -95,6 +97,22 @@ _BUILD_TEST_LOCATOR_CANDIDATE_MAX_ITEMS = 6
_BUILD_TEST_LOCATOR_SELECTOR_MAX_CHARS = 240
_BUILD_TEST_OBSTRUCTION_VALUE_MAX_CHARS = 240
_BUILD_TEST_IDENTITY_LABEL_MAX_CHARS = 2_048
+_LOOP_INPUT_MAX_CHARS = {
+ "workflow_run_id": _BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ "workflow_id": _BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ "workflow_permanent_id": _BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ "loops[].block_label": _BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ "loops[].enclosing_loop_label": _BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ "loops[].loop_over_parameter_key": _BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ "loops[].producer_block_label": _BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ "loops[].producer_output_parameter_id": _BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ "loops[].loop_variable_reference": 200,
+}
+# The exact notices bounded_loop_inputs emits; the handoff cannot re-derive them from already-shortened values.
+LOOP_INPUT_OMISSION_NOTICES = frozenset(
+ f"loop_inputs.{field_name} shortened at {max_chars} characters."
+ for field_name, max_chars in _LOOP_INPUT_MAX_CHARS.items()
+)
_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
_JPEG_PREFIX = b"\xff\xd8\xff"
@@ -787,6 +805,40 @@ def _bounded_packet_page_state(
return page_state.model_copy(update=updates)
+def bounded_loop_inputs(loop_inputs: LoopInputs, notices: list[str]) -> LoopInputs:
+ def bounded(value: str | None, field_name: str) -> str | None:
+ return _bounded_packet_string(
+ value,
+ field_name=f"loop_inputs.{field_name}",
+ max_chars=_LOOP_INPUT_MAX_CHARS[field_name],
+ notices=notices,
+ )
+
+ loops = [
+ loop.model_copy(
+ update={
+ "block_label": bounded(loop.block_label, "loops[].block_label"),
+ "enclosing_loop_label": bounded(loop.enclosing_loop_label, "loops[].enclosing_loop_label"),
+ "loop_over_parameter_key": bounded(loop.loop_over_parameter_key, "loops[].loop_over_parameter_key"),
+ "producer_block_label": bounded(loop.producer_block_label, "loops[].producer_block_label"),
+ "producer_output_parameter_id": bounded(
+ loop.producer_output_parameter_id, "loops[].producer_output_parameter_id"
+ ),
+ "loop_variable_reference": bounded(loop.loop_variable_reference, "loops[].loop_variable_reference"),
+ }
+ )
+ for loop in loop_inputs.loops
+ ]
+ return loop_inputs.model_copy(
+ update={
+ "workflow_run_id": bounded(loop_inputs.workflow_run_id, "workflow_run_id"),
+ "workflow_id": bounded(loop_inputs.workflow_id, "workflow_id"),
+ "workflow_permanent_id": bounded(loop_inputs.workflow_permanent_id, "workflow_permanent_id"),
+ "loops": loops,
+ }
+ )
+
+
def _compact_block_fact_maps(packet: BuildTestEvidencePacket, notices: list[str]) -> BuildTestEvidencePacket:
return packet.model_copy(
update={
@@ -811,6 +863,20 @@ def _compact_block_fact_maps(packet: BuildTestEvidencePacket, notices: list[str]
)
+def _compact_loop_inputs(packet: BuildTestEvidencePacket, notices: list[str]) -> BuildTestEvidencePacket:
+ loop_inputs = packet.run.loop_inputs
+ if not isinstance(loop_inputs, LoopInputs):
+ return packet
+ loops = _compacted(loop_inputs.loops, 6, field_name="run.loop_inputs.loops", notices=notices, keep="first")
+ if not (dropped := len(loop_inputs.loops) - len(loops)):
+ return packet
+ loops_omitted = (loop_inputs.loops_omitted or 0) + dropped
+ compacted = loop_inputs.model_copy(update={"loops": loops, "loops_omitted": loops_omitted})
+ return packet.model_copy(
+ update={"run": packet.run.model_copy(update={"loop_inputs": compacted}), "omission_notices": notices}
+ )
+
+
def _clipped_120(text: str | None) -> str | None:
return text[:117] + "..." if text is not None and len(text) > 120 else text
@@ -1356,6 +1422,11 @@ def project_build_test_packet_for_llm(packet: BuildTestEvidencePacket) -> BuildT
max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
notices=notices,
),
+ "loop_inputs": (
+ bounded_loop_inputs(loop_inputs, notices)
+ if isinstance(loop_inputs := projected.run.loop_inputs, LoopInputs)
+ else loop_inputs
+ ),
}
)
screenshot = projected.screenshot.model_copy(
@@ -1375,6 +1446,9 @@ def project_build_test_packet_for_llm(packet: BuildTestEvidencePacket) -> BuildT
# Repeated per-block facts give way before the workflow readback the repair turn needs more.
projected = _compact_block_fact_maps(projected, notices)
serialized = json.dumps(projected.model_dump(mode="json", exclude_none=True), ensure_ascii=False)
+ if len(serialized) > _BUILD_TEST_PACKET_MAX_CHARS:
+ projected = _compact_loop_inputs(projected, notices)
+ serialized = json.dumps(projected.model_dump(mode="json", exclude_none=True), ensure_ascii=False)
if len(serialized) > _BUILD_TEST_PACKET_MAX_CHARS and projected.canonical_workflow_yaml is not None:
excess = len(serialized) - _BUILD_TEST_PACKET_MAX_CHARS
retained_chars = max(0, len(projected.canonical_workflow_yaml) - excess - 200)
@@ -1404,11 +1478,12 @@ def project_build_test_packet_for_llm(packet: BuildTestEvidencePacket) -> BuildT
def project_direct_test_handoff_packet_for_llm(packet: BuildTestEvidencePacket) -> BuildTestEvidencePacket:
- # This handoff rebuilds its own notices, so it carries forward only the mint-time fact it cannot
+ # This handoff rebuilds its own notices, so it carries forward only the mint-time facts it cannot
# reconstruct; adopting the recorded list wholesale would relay run-supplied prose to the model.
notices: list[str] = []
if OBSERVED_BLOCK_END_URLS_WITHHELD in packet.omission_notices:
notices.append(OBSERVED_BLOCK_END_URLS_WITHHELD)
+ notices.extend(notice for notice in packet.omission_notices if notice in LOOP_INPUT_OMISSION_NOTICES)
if packet.canonical_workflow_yaml is None:
notices.append("canonical_workflow_yaml omitted: no persisted workflow readback was recorded.")
if packet.run.workflow_run_id is None:
@@ -1672,6 +1747,8 @@ def sanitize_tool_result_for_llm(tool_name: str, result: dict[str, Any]) -> dict
# and values registered for secret scrubbing during finalization.
data.pop("action_observations", None)
data.pop("action_trace_summary", None)
+ data.pop(LOOP_INPUTS_KEY, None)
+ data.pop("loop_inputs_omission_notices", None)
data.pop("observed_block_end_urls", None)
data.pop("per_block_action_observations", None)
data.pop("block_fact_omission_notices", None)
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index 2b12515bc..efc4b1b44 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -10,6 +10,7 @@ from enum import StrEnum
from typing import Any
import structlog
+import yaml
from pydantic import JsonValue
from skyvern.cli.core.js_dispatch import outer_cap_seconds
@@ -553,6 +554,77 @@ PUBLISH_FILE_HELPER_CONTRACT: dict[str, Any] = {
}
+_FOR_LOOP_PROPERTY_DESCRIPTIONS = {
+ "loop_over_parameter_key": (
+ "A workflow parameter key, or an earlier block's `<label>_output` when that output is itself the list. "
+ "To loop over an earlier block's output, set this key and leave `loop_variable_reference` unset. "
+ "Ignored whenever `loop_variable_reference` is non-empty."
+ ),
+ "loop_variable_reference": (
+ "A block label (also tried as `<label>.extracted_information`, `<label>.extracted_information.results` "
+ "and `<label>.results`), dotted path or template expression for the list. When non-empty it overrides "
+ "`loop_over_parameter_key`, even when both are set; a value that resolves to nothing is read as a "
+ "natural-language description of the list. `current_value` exists only inside a running for_loop's "
+ "`loop_blocks` (a while_loop leaves it unset), so name it here only in a loop nested inside an outer "
+ "for_loop."
+ ),
+}
+
+_FOR_LOOP_EXAMPLE: dict[str, JsonValue] = {
+ "parameters": [{"parameter_type": "workflow", "key": "rows_file", "workflow_parameter_type": "file_url"}],
+ "blocks": [
+ {
+ "block_type": "file_url_parser",
+ "label": "parse_rows",
+ "file_url": "{{ rows_file }}",
+ "file_type": "csv",
+ "next_block_label": "visit_each_row",
+ },
+ {
+ "block_type": "for_loop",
+ "label": "visit_each_row",
+ "loop_over_parameter_key": "parse_rows_output",
+ "loop_blocks": [{"block_type": "goto_url", "label": "open_row_url", "url": "{{ current_value.url }}"}],
+ },
+ ],
+}
+
+
+_FOR_LOOP_GUIDANCE = "\n".join(
+ [
+ "Purpose: Iterate over a list and run `loop_blocks` once per item. A for_loop reads exactly one input.",
+ "",
+ f"- loop_over_parameter_key: {_FOR_LOOP_PROPERTY_DESCRIPTIONS['loop_over_parameter_key']}",
+ f"- loop_variable_reference: {_FOR_LOOP_PROPERTY_DESCRIPTIONS['loop_variable_reference']}",
+ (
+ "- loop_blocks (required): the blocks run for each item, chained with next_block_label. Inside them "
+ "{{ current_value }} (also {{ current_item }}) is the current item and {{ current_index }} its position."
+ ),
+ "- complete_if_empty: true completes the loop successfully when the list is empty.",
+ "",
+ "Example - loop over a parser's rows (the parser's output is the list):",
+ yaml.safe_dump(_FOR_LOOP_EXAMPLE, sort_keys=False).rstrip(),
+ ]
+)
+
+
+def _apply_for_loop_schema_guidance(data: dict[str, Any]) -> None:
+ schema = data.get("schema")
+ if (
+ isinstance(schema, dict)
+ and isinstance(ref := schema.get("$ref"), str)
+ and isinstance(schema.get("$defs"), dict)
+ ):
+ # ForLoopBlockYAML nests blocks recursively, so its JSON schema is a $ref into $defs.
+ schema = schema["$defs"].get(ref.removeprefix("#/$defs/"))
+ properties = schema.get("properties") if isinstance(schema, dict) else None
+ if isinstance(properties, dict):
+ for name, description in _FOR_LOOP_PROPERTY_DESCRIPTIONS.items():
+ if isinstance(properties.get(name), dict):
+ properties[name] = {**properties[name], "description": description}
+ data["example"] = copy.deepcopy(_FOR_LOOP_EXAMPLE)
+
+
async def _get_block_schema_post_hook(
result: dict[str, Any],
raw: dict[str, Any],
@@ -563,6 +635,8 @@ async def _get_block_schema_post_hook(
capability = _copilot_authoring_capability(ctx)
data = result.get("data")
if isinstance(data, dict):
+ if data.get("block_type") == "for_loop":
+ _apply_for_loop_schema_guidance(data)
block_types = data.get("block_types")
if isinstance(block_types, dict):
for banned in _copilot_banned_block_types(ctx):
@@ -686,12 +760,16 @@ async def _get_workflow_knowledge_post_hook(
) -> dict[str, Any]:
capability = _copilot_authoring_capability(ctx)
data = result.get("data")
- if isinstance(data, dict) and capability.code_blocks and capability.agent_blocks:
- sections = data.get("sections")
- if isinstance(sections, dict):
- choosing = sections.get("choosing_a_block")
- if isinstance(choosing, dict) and isinstance(choosing.get("content"), str):
- choosing["content"] = f"{AUTHORING_FAMILY_GUIDANCE}\n\n{choosing['content']}"
+ sections = data.get("sections") if isinstance(data, dict) else None
+ if not isinstance(sections, dict):
+ return result
+ for_loop = sections.get("for_loop_block")
+ if isinstance(for_loop, dict):
+ for_loop["content"] = _FOR_LOOP_GUIDANCE
+ if capability.code_blocks and capability.agent_blocks:
+ choosing = sections.get("choosing_a_block")
+ if isinstance(choosing, dict) and isinstance(choosing.get("content"), str):
+ choosing["content"] = f"{AUTHORING_FAMILY_GUIDANCE}\n\n{choosing['content']}"
return result
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index dd9a2e961..6f1eeebb5 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -64,6 +64,10 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestPacketRunBrowser,
BuildTestPacketScreenshot,
BuildTestPacketUnfinishedItem,
+ LoopInputFact,
+ LoopInputs,
+ LoopInputsUnavailable,
+ LoopSelectedInput,
RecordedBuildTestOutcome,
SolverAttempt,
append_omission_notice,
@@ -141,6 +145,8 @@ from skyvern.forge.sdk.copilot.output_utils import (
_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY,
_INTERNAL_RUN_OUTCOME_RECORDED_KEY,
BUILD_TEST_PACKET_KEY,
+ LOOP_INPUTS_KEY,
+ bounded_loop_inputs,
build_run_blocks_response,
iter_failure_reasons,
labelled_block_names,
@@ -240,7 +246,13 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.utils.pdf_parser import extract_pdf_file
-from skyvern.forge.sdk.workflow.models.block import CodeBlock, get_all_blocks
+from skyvern.forge.sdk.workflow.models.block import (
+ BlockTypeVar,
+ CodeBlock,
+ ForLoopBlock,
+ WhileLoopBlock,
+ get_all_blocks,
+)
from skyvern.forge.sdk.workflow.models.code_block_recorder import RECORDED_FAILURE_RESPONSE_MAX_CHARS
from skyvern.forge.sdk.workflow.models.parameter import (
OutputParameter,
@@ -249,6 +261,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
is_sensitive_workflow_parameter,
)
from skyvern.forge.sdk.workflow.models.workflow import (
+ COPILOT_TEST_WORKFLOW_CREATOR,
Workflow,
WorkflowRun,
WorkflowRunOutputParameter,
@@ -1775,6 +1788,138 @@ def _execution_source_facts(
}
+_LOOP_INPUTS_MAX_LOOPS = 20
+_LOOP_VALUES_COUNTS_MAX_ROWS = 10
+
+
+def loop_selected_input(block: ForLoopBlock) -> LoopSelectedInput:
+ """Mirrors ForLoopBlock.get_loop_over_parameter_values: a truthy reference wins over loop_over."""
+ if block.loop_variable_reference:
+ return "loop_variable_reference"
+ if block.loop_over is not None:
+ return "loop_over_parameter_key"
+ return "none"
+
+
+def loop_definition_facts(workflow: Workflow) -> LoopInputs | None:
+ producers_by_id, producers_by_key = _workflow_output_parameter_indexes(workflow)
+ loops: list[LoopInputFact] = []
+
+ def visit(blocks: Sequence[BlockTypeVar], enclosing_loop_label: str | None) -> None:
+ for block in blocks:
+ if isinstance(block, ForLoopBlock):
+ loop_over = block.loop_over
+ reference = block.loop_variable_reference
+ # The converter replaces loop_over with the parameter a reference names, losing the authored key.
+ if loop_over is not None and reference and loop_over.key == reference.strip(" {}"):
+ loop_over = None
+ # A loop_over saved by an older version keeps that version's id; the executor resolves it by key.
+ producer = (
+ producers_by_id.get(loop_over.output_parameter_id) or producers_by_key.get(loop_over.key, {})
+ if isinstance(loop_over, OutputParameter)
+ else {}
+ )
+ loops.append(
+ LoopInputFact(
+ block_label=block.label,
+ enclosing_loop_label=enclosing_loop_label,
+ loop_over_parameter_key=loop_over.key if loop_over is not None else None,
+ producer_block_label=producer.get("block_label"),
+ producer_output_parameter_id=producer.get("output_parameter_id"),
+ loop_variable_reference=reference,
+ selected_input=loop_selected_input(block),
+ )
+ )
+ if isinstance(block, (ForLoopBlock, WhileLoopBlock)):
+ visit(block.loop_blocks, block.label)
+
+ visit(workflow.workflow_definition.blocks, None)
+ if not loops:
+ return None
+ return LoopInputs(
+ workflow_id=workflow.workflow_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ version=workflow.version,
+ loops=loops[:_LOOP_INPUTS_MAX_LOOPS],
+ loops_omitted=len(loops) - _LOOP_INPUTS_MAX_LOOPS if len(loops) > _LOOP_INPUTS_MAX_LOOPS else None,
+ )
+
+
+def with_loop_observations(
+ definitions: LoopInputs, workflow_run_id: str, rows: Sequence[WorkflowRunBlock]
+) -> LoopInputs:
+ """A row whose loop_values was never written reports not_recorded; a loop with no row in this run (never
+ reached, or seeded from an earlier run) carries no observations."""
+ loops: list[LoopInputFact] = []
+ for loop in definitions.loops:
+ loop_rows = [row for row in rows if row.block_type == BlockType.FOR_LOOP and row.label == loop.block_label]
+ if not loop_rows:
+ loops.append(loop)
+ continue
+ counts: list[int | Literal["not_recorded"]] = [
+ len(row.loop_values) if row.loop_values is not None else "not_recorded" for row in loop_rows
+ ]
+ omitted = len(counts) - _LOOP_VALUES_COUNTS_MAX_ROWS
+ loops.append(
+ loop.model_copy(
+ update={
+ "run_rows": len(loop_rows),
+ "loop_values_counts": counts[-_LOOP_VALUES_COUNTS_MAX_ROWS:],
+ "loop_values_counts_omitted": omitted if omitted > 0 else None,
+ }
+ )
+ )
+ return definitions.model_copy(update={"workflow_run_id": workflow_run_id, "loops": loops})
+
+
+def _loop_definitions_or_unavailable(workflow: Workflow) -> LoopInputs | LoopInputsUnavailable | None:
+ try:
+ return loop_definition_facts(workflow)
+ except Exception:
+ LOG.warning("Loop input definition facts unavailable", workflow_id=workflow.workflow_id, exc_info=True)
+ return LoopInputsUnavailable()
+
+
+def _observed_loop_inputs(
+ definitions: LoopInputs | LoopInputsUnavailable, workflow_run_id: str, rows: Sequence[WorkflowRunBlock]
+) -> tuple[LoopInputs | LoopInputsUnavailable, list[str]]:
+ if isinstance(definitions, LoopInputsUnavailable):
+ return definitions, []
+ notices: list[str] = []
+ try:
+ return bounded_loop_inputs(with_loop_observations(definitions, workflow_run_id, rows), notices), notices
+ except Exception:
+ LOG.warning("Loop input observations unavailable", workflow_run_id=workflow_run_id, exc_info=True)
+ return LoopInputsUnavailable(), []
+
+
+def _attach_loop_inputs(
+ data: dict[str, Any], execution: _RunExecution, workflow_run_id: str, rows: Sequence[WorkflowRunBlock]
+) -> None:
+ if execution.loop_definitions is None:
+ return
+ loop_inputs, notices = _observed_loop_inputs(execution.loop_definitions, workflow_run_id, rows)
+ data[LOOP_INPUTS_KEY] = loop_inputs.model_dump(mode="json", exclude_none=True)
+ if notices:
+ data["loop_inputs_omission_notices"] = notices
+
+
+def _read_run_loop_inputs(
+ run_workflow: Workflow | None, run_created_at: datetime, workflow_run_id: str, rows: Sequence[WorkflowRunBlock]
+) -> tuple[LoopInputs | LoopInputsUnavailable | None, list[str]]:
+ loop_definitions = _loop_definitions_or_unavailable(run_workflow) if run_workflow is not None else None
+ # A saved version can be overwritten in place after the run; a test version is only ever soft-deleted.
+ if run_workflow is None or (
+ run_workflow.created_by != COPILOT_TEST_WORKFLOW_CREATOR
+ and _as_utc(run_workflow.modified_at) > _as_utc(run_created_at)
+ ):
+ ran_loop = loop_definitions is not None or any(row.block_type == BlockType.FOR_LOOP for row in rows)
+ return (LoopInputsUnavailable() if ran_loop else None), []
+ if loop_definitions is None:
+ return None, []
+ return _observed_loop_inputs(loop_definitions, workflow_run_id, rows)
+
+
@dataclass(frozen=True)
class CopilotExecutionSnapshot:
"""One provenance-closed workflow definition and the rows a run binds against."""
@@ -1812,6 +1957,7 @@ class _RunExecution:
dispatched_to_worker: bool = False
dispatched_input_values: dict[str, Any] = dataclass_field(default_factory=dict, repr=False)
recorded_settings: OriginExecutionSettings | None = dataclass_field(default=None, repr=False)
+ loop_definitions: LoopInputs | LoopInputsUnavailable | None = None
def __post_init__(self) -> None:
# Runtime blocks render templates in place. The receipt must keep the authored
@@ -4033,6 +4179,8 @@ async def _run_blocks_and_collect_debug(
if dispatch_workflow is not None:
snapshot = _materialized_execution_snapshot(snapshot, dispatch_workflow)
+ # Read before execution: a natural-language loop reference is rewritten on the block while it runs.
+ execution.loop_definitions = _loop_definitions_or_unavailable(snapshot.workflow)
if execution.recorded_settings is not None:
# Persistence owns model/header/profile-key settings; the attached browser owns
@@ -4592,6 +4740,7 @@ async def _run_blocks_and_collect_debug(
run_ok=False,
page_evidence=_same_run_page_evidence_for_result(ctx, workflow_run.workflow_run_id),
)
+ _attach_loop_inputs(result["data"], execution, workflow_run.workflow_run_id, watchdog_block_rows)
_attach_block_fact_projection(
result["data"],
watchdog_block_rows,
@@ -4777,6 +4926,7 @@ async def _run_blocks_and_collect_debug(
result_data["failure_reason"] = redact_totp_runtime_values(run.failure_reason)
if not run_ok and run and getattr(run, "failure_category", None):
result_data["failure_category"] = run.failure_category
+ _attach_loop_inputs(result_data, execution, workflow_run.workflow_run_id, run_block_rows)
_attach_block_fact_projection(
result_data,
run_block_rows,
@@ -5127,6 +5277,11 @@ async def _get_run_results(
}
if run_workflow is not None and workflow_run_id == ctx.proposal_workflow_run_id:
result_data["execution_source"] = _execution_source_facts(run_workflow, provenance="staged")
+ loop_inputs, loop_input_notices = _read_run_loop_inputs(run_workflow, run.created_at, workflow_run_id, blocks)
+ if loop_inputs is not None:
+ result_data[LOOP_INPUTS_KEY] = loop_inputs.model_dump(mode="json", exclude_none=True)
+ if loop_input_notices:
+ result_data["loop_inputs_omission_notices"] = loop_input_notices
_attach_block_fact_projection(
result_data,
blocks,
@@ -7180,6 +7335,7 @@ def build_test_evidence_packet(
if not action_observations:
omission_notices.append(ACTION_OBSERVATIONS_EMPTY)
omission_notices.extend(_packet_string_list(data.get("block_fact_omission_notices")))
+ omission_notices.extend(_packet_string_list(data.get("loop_inputs_omission_notices")))
observed_block_end_urls = coerce_block_end_urls(data.get("observed_block_end_urls"), omission_notices)
per_block_action_observations = coerce_block_action_observations(
data.get("per_block_action_observations"), omission_notices
@@ -7327,6 +7483,7 @@ def build_test_evidence_packet(
executed_block_labels=executed_labels,
run=BuildTestPacketRun(
execution_source=data.get("execution_source"),
+ loop_inputs=data.get(LOOP_INPUTS_KEY),
browser_start=data.get("browser_start"),
workflow_run_id=run_id,
browser_session_id=_packet_string(data.get("browser_session_id")),
diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.py
index 5c794d1e2..96e8f9045 100644
--- a/skyvern/forge/sdk/event/factory.py
+++ b/skyvern/forge/sdk/event/factory.py
@@ -39,7 +39,6 @@ class EventStrategyFactory:
__cursor_vis_cache: CursorEventStrategy | None = None
__input: InputEventStrategy | None = None
__scroll: ScrollEventStrategy | None = None
- __profile: str | None = None
__metrics: _EventMetrics = _EventMetrics()
# -- setters ----------------------------------------------------------------
@@ -63,22 +62,8 @@ class EventStrategyFactory:
EventStrategyFactory.__cursor_vis_cache = None
EventStrategyFactory.__input = None
EventStrategyFactory.__scroll = None
- EventStrategyFactory.__profile = None
EventStrategyFactory.__metrics = _EventMetrics()
- @staticmethod
- def record_profile(profile: str) -> None:
- """Name the profile whoever registered the current strategies registered them under."""
- EventStrategyFactory.__profile = profile
-
- @staticmethod
- def registered_profile() -> str:
- """The recorded profile; "none" when nothing is registered, "unrecorded" when strategies were set unnamed."""
- if EventStrategyFactory.__profile is not None:
- return EventStrategyFactory.__profile
- strategies = (EventStrategyFactory.__cursor, EventStrategyFactory.__input, EventStrategyFactory.__scroll)
- return "none" if all(strategy is None for strategy in strategies) else "unrecorded"
-
# -- getters (always return a non-None strategy) ----------------------------
@staticmethod
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index 35e45073f..8f60bb070 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -37,7 +37,6 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTask
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.workflow.models.credential_release import CredentialReleaseGuard
-from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3.code_surface import apply_surface, configured_surface
from skyvern.forge.taskv3.goal_check import (
GOAL_CHECK_TIMEOUT_SECONDS,
@@ -712,7 +711,6 @@ async def run_task_v3_agent_loop(
# The run's model, so exposure rates on this line split per model like the re-ask line's.
llm_key=llm_caller.llm_key,
unlisted_reask=outcome.unlisted_reask,
- **input_dispatch.arm_fields(),
# The loop's progress signals ride here rather than on records of their own: this line
# already fires exactly once per run and already carries block_type, so collapsing removes a
# per-run indexed event and makes the join to block_type free instead of a second lookup.
diff --git a/skyvern/forge/taskv3/input_dispatch.py b/skyvern/forge/taskv3/input_dispatch.py
index 663ddf3ec..bd493aab1 100644
--- a/skyvern/forge/taskv3/input_dispatch.py
+++ b/skyvern/forge/taskv3/input_dispatch.py
@@ -1,109 +1,23 @@
"""The one place Task V3 sends pointer and keyboard input to a page.
-Control runs the Playwright call each tool made before this module existed, argument for argument. Treatment
-sends a gesture through EventStrategyFactory wherever v1 does and calls Playwright directly wherever v1 does, so
-the run follows whatever USE_EVENT_STRATEGIES registered for the process, like a v1 block of the same run.
+Each gesture runs the Playwright call each tool made before this module existed, argument for argument.
"""
from __future__ import annotations
-import asyncio
from typing import Any, TypeAlias, cast
-import structlog
from playwright.async_api import ElementHandle, FileChooser, Frame, Locator, Page
-from playwright.async_api import TimeoutError as PlaywrightTimeoutError
-
-from skyvern.config import settings
-from skyvern.constants import TEXT_PRESS_MAX_LENGTH
-from skyvern.forge.sdk.event.factory import EventStrategyFactory
-from skyvern.forge.taskv3.run_arms import HUMANIZED_INPUT_FLAG, current_run_arm, run_arm_enabled
-from skyvern.webeye.browser_object_predicates import is_page_like
-
-LOG = structlog.get_logger()
Realm: TypeAlias = Page | Frame
Target: TypeAlias = str | Locator | ElementHandle
-# After the cursor reaches a target, wait until the page has painted twice and then gone this long without a DOM
-# mutation, so a hover-intent menu crossed on the way has finished opening or closing before a caller takes its
-# pre-action baseline. Capped: a page that never stops mutating (a ticker, a spinner) costs the cap, not a stall.
-_SETTLE_QUIET_MS = 150
-_SETTLE_CAP_MS = 1000
-_SETTLE_JS = r"""(arg) => new Promise((resolve) => {
- const started = performance.now();
- let last = started;
- const obs = new MutationObserver(() => { last = performance.now(); });
- const tick = () => {
- const now = performance.now();
- if (now - last >= arg.quietMs || now - started >= arg.capMs) { obs.disconnect(); resolve(true); return; }
- setTimeout(tick, 25);
- };
- requestAnimationFrame(() => requestAnimationFrame(() => {
- obs.observe(document.documentElement, { subtree: true, childList: true, attributes: true, characterData: true });
- last = performance.now();
- setTimeout(tick, 25);
- }));
- setTimeout(() => { obs.disconnect(); resolve(false); }, arg.capMs);
-})"""
-
-
-def arm_fields() -> dict[str, Any]:
- return {
- "humanized_input_arm": current_run_arm(HUMANIZED_INPUT_FLAG),
- "humanized_input_in_effect": humanized(),
- "global_event_strategy_profile": EventStrategyFactory.registered_profile(),
- }
-
-
-def humanized() -> bool:
- return run_arm_enabled(HUMANIZED_INPUT_FLAG, settings.TASK_V3_HUMANIZED_INPUT)
-
def _opts(**options: Any) -> dict[str, Any]:
- # Only what the caller passed, so a control call carries exactly the keywords the tool always sent.
+ # Only what the caller passed, so each call carries exactly the keywords the tool always sent.
return {key: value for key, value in options.items() if value is not None}
-def _page_of(realm: Realm) -> Page:
- # A Frame has no mouse or keyboard; the factory always drives the tab that owns it.
- return cast(Page, realm) if is_page_like(realm) else cast(Frame, realm).page
-
-
-def _locator(realm: Realm, target: str | Locator) -> Locator:
- # A page-level selector method acts on the first match; a locator is strict unless told otherwise.
- return realm.locator(target).first if isinstance(target, str) else target
-
-
-async def _move_to_element(page: Page, element: ElementHandle) -> None:
- try:
- box = await element.bounding_box()
- if box is None:
- return
- await EventStrategyFactory.move_cursor(page, box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)
- except Exception:
- LOG.debug("taskv3 cursor move to a handed-in element failed, clicking anyway", exc_info=True)
-
-
-async def approach(realm: Realm, target: str | Locator) -> None:
- """Treatment: bring the cursor to ``target`` and let the page settle, so travel effects predate a baseline."""
- if not humanized():
- return
- locator = _locator(realm, target)
- try:
- # The cursor is clamped to the viewport, so an off-screen target would otherwise be scrolled to, and
- # travelled to again, only after the baseline.
- await locator.scroll_into_view_if_needed(timeout=2000)
- except Exception:
- LOG.debug("taskv3 scroll before cursor approach failed", exc_info=True)
- await EventStrategyFactory.move_to_element(locator.page, locator)
- try:
- async with asyncio.timeout((_SETTLE_CAP_MS + 500) / 1000):
- await realm.evaluate(_SETTLE_JS, {"quietMs": _SETTLE_QUIET_MS, "capMs": _SETTLE_CAP_MS})
- except Exception:
- LOG.debug("taskv3 settle after cursor approach did not complete", exc_info=True)
-
-
async def click(
realm: Realm,
target: str | Locator,
@@ -113,21 +27,10 @@ async def click(
position: dict[str, float] | None = None,
) -> None:
options = _opts(timeout=timeout, force=force, position=position)
- if not humanized():
- if isinstance(target, str):
- await realm.click(target, **options)
- else:
- await target.click(**options)
- return
- locator = _locator(realm, target)
- if force is not None or position is not None:
- await EventStrategyFactory.move_to_element(locator.page, locator)
- if isinstance(target, str):
- await realm.click(target, **options)
- else:
- await target.click(**options)
- return
- await EventStrategyFactory.click_element(locator.page, locator, timeout=timeout)
+ if isinstance(target, str):
+ await realm.click(target, **options)
+ else:
+ await target.click(**options)
async def click_handle(
@@ -138,27 +41,19 @@ async def click_handle(
force: bool | None = None,
position: dict[str, float] | None = None,
) -> None:
- if humanized():
- # v1's pattern for a click the cursor strategy cannot express: move there, then click directly.
- await _move_to_element(_page_of(realm), handle)
await handle.click(**_opts(timeout=timeout, force=force, position=position))
async def click_at(page: Page, x: float, y: float) -> None:
- if humanized():
- await EventStrategyFactory.move_cursor(page, x, y)
await page.mouse.click(x, y)
async def hover(realm: Realm, selector: str, *, timeout: float | None = None) -> None:
- if humanized():
- locator = _locator(realm, selector)
- await EventStrategyFactory.move_to_element(locator.page, locator)
await realm.hover(selector, **_opts(timeout=timeout))
async def js_click(realm: Realm, root_query_js: str, arg: Any) -> Any:
- """Click a hidden native control from inside the page. No pointer events, in either arm, as in v1."""
+ """Click a hidden native control from inside the page. No pointer events, as in v1."""
return await realm.evaluate(
"(arg) => { const _q = "
+ root_query_js
@@ -176,7 +71,6 @@ async def focus(realm: Realm, target: str | Locator, *, timeout: float | None =
async def fill(realm: Realm, target: Target, value: str, *, timeout: float | None = None) -> None:
- """An atomic write, in both arms: v1 fills plain fields and secrets in one call and never humanizes them."""
if isinstance(target, str):
await realm.fill(target, value, **_opts(timeout=timeout))
else:
@@ -184,11 +78,7 @@ async def fill(realm: Realm, target: Target, value: str, *, timeout: float | Non
async def clear(realm: Realm, selector: str, *, timeout: float | None = None) -> None:
- if not humanized():
- await realm.fill(selector, "", **_opts(timeout=timeout))
- return
- locator = _locator(realm, selector)
- await EventStrategyFactory.clear_field(locator.page, locator, 0, timeout=timeout)
+ await realm.fill(selector, "", **_opts(timeout=timeout))
async def type_keys(
@@ -198,50 +88,22 @@ async def type_keys(
*,
delay: float | None = None,
timeout: float | None = None,
- secret: bool = False,
- replace: bool = False,
) -> None:
"""Key events for ``text``. ``target`` None types into whatever holds focus, with no focus of its own;
- ``realm`` is then the page.
-
- A secret is typed key by key as before in both arms: a humanized keyboard mistypes and backspaces, and a
- fill would not advance split code boxes. ``replace`` says the caller has just emptied the field, so
- treatment may write the untyped part with fill."""
+ ``realm`` is then the page."""
options = _opts(delay=delay, timeout=timeout)
- if secret or not humanized():
- if target is None:
- await cast(Page, realm).keyboard.type(text, **options)
- elif isinstance(target, str):
- await realm.type(target, text, **options)
- else:
- await target.press_sequentially(text, **options)
- return
- # As v1 types: all but the last few characters in one write, the tail through the registered strategy,
- # which owns the cadence. An emptied field takes the head as a fill; one being appended to takes it as
- # inserted text at the caret, since a fill would replace what the field already holds.
- cut = max(len(text) - TEXT_PRESS_MAX_LENGTH, 0)
- head, tail = text[:cut], text[cut:]
- page = cast(Page, realm) if target is None else _page_of(realm)
- seconds = None if not timeout else timeout / 1000
- try:
- async with asyncio.timeout(seconds):
- if head and replace and target is not None:
- await fill(realm, target, head, timeout=timeout)
- elif target is not None:
- # Focused once: a humanized keyboard given a locator re-focuses the field before every key,
- # which puts the caret back at the start of a select-on-focus field.
- await focus(realm, target, timeout=timeout)
- if head and not (replace and target is not None):
- await page.keyboard.insert_text(head)
- await EventStrategyFactory.type_text(page, None, tail, timeout=timeout)
- except TimeoutError as exc:
- raise PlaywrightTimeoutError(f"Timeout {timeout}ms exceeded.") from exc
+ if target is None:
+ await cast(Page, realm).keyboard.type(text, **options)
+ elif isinstance(target, str):
+ await realm.type(target, text, **options)
+ else:
+ await target.press_sequentially(text, **options)
async def press(
realm: Realm, target: str | Locator | ElementHandle | None, key: str, *, timeout: float | None = None
) -> None:
- """A key press, raw in both arms as v1's keypress is. ``target`` None presses on ``realm``, the page's, keyboard."""
+ """A key press, raw as v1's keypress is. ``target`` None presses on ``realm``, the page's, keyboard."""
options = _opts(timeout=timeout)
if target is None:
await cast(Page, realm).keyboard.press(key, **options)
@@ -275,7 +137,4 @@ async def set_input_files(target: Locator | ElementHandle, paths: list[str]) ->
async def wheel(page: Page, delta_x: float, delta_y: float) -> None:
- if humanized():
- await EventStrategyFactory.scroll_by(page, delta_x, delta_y)
- else:
- await page.mouse.wheel(delta_x, delta_y)
+ await page.mouse.wheel(delta_x, delta_y)
diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.py
index c754193f0..86ff96cea 100644
--- a/skyvern/forge/taskv3/run_arms.py
+++ b/skyvern/forge/taskv3/run_arms.py
@@ -20,7 +20,6 @@ DATE_SEGMENT_AIM_FLAG = "TASK_V3_DATE_SEGMENT_AIM"
EXTRACTION_REPORTS_FLAG = "TASK_V3_EXTRACTION_REPORTS"
GOAL_CHECK_FLAG = "TASK_V3_GOAL_CHECK"
GOAL_CHECK_ENFORCE_FLAG = "TASK_V3_GOAL_CHECK_ENFORCE"
-HUMANIZED_INPUT_FLAG = "TASK_V3_HUMANIZED_INPUT"
# Person properties a flag is evaluated with beyond organization_id; resolve_run_arm drops any other key. The
# PostHog preflight (scripts/check_run_arm_flags.py) reads this mapping to accept release conditions on them.
@@ -96,12 +95,3 @@ def run_arm_enabled(flag: str, forced: bool) -> bool:
return False
pin = context.run_arms.get(flag)
return pin is not None and pin[1] == "treatment"
-
-
-def current_run_arm(flag: str) -> RunArm | None:
- """The arm pinned for ``flag`` on the current run, or None when it was never resolved."""
- context = skyvern_context.current()
- if context is None:
- return None
- pin = context.run_arms.get(flag)
- return pin[1] if pin is not None else None
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 1f3218558..b387c194e 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -13698,7 +13698,6 @@ def build_browser_tools(
)
else:
await _require_single_target(page, selector)
- await input_dispatch.approach(page, selector)
pre: dict[str, Any] | None = None
try:
pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _probe_arg(page, selector))
@@ -14430,7 +14429,6 @@ def build_browser_tools(
# held value, and only with its own list known closed; every other field acts and verifies the click.
if not settings.TASK_V3_SEMANTIC_COMMIT_VERIFY:
return False
- await input_dispatch.approach(page, selector)
try:
if not await page.evaluate(_OWN_LIST_CLOSED_JS, await _probe_arg(page, selector)):
return False
@@ -15221,7 +15219,6 @@ def build_browser_tools(
collateral: list[list[str]] | None = None,
query: str | None = None,
press_enter: bool = False,
- secret: bool = False,
) -> tuple[_TypeaheadPick, str | None, str | None, _Reach]:
# Keystroke-type (so a widget's async suggestion fetch fires on real key events). Snapshot the
# visible DOM BEFORE the focus click, not just before typing: a widget that opens its full list on
@@ -15236,7 +15233,6 @@ def build_browser_tools(
# Same reason: fill()/type() below open the widget's own list (aria-expanded=true) for the rest
# of this attempt, and ownCommittedSurface reads nothing while it is open.
pre_own = await _own_surface_text(page, selector)
- await input_dispatch.approach(page, selector)
try:
await page.evaluate(_PRESNAPSHOT_JS)
except Exception:
@@ -15267,9 +15263,7 @@ def build_browser_tools(
# Clearing an empty multi-value input sends Delete, which removes the field's last chip.
if not (pre_value == "" and await _holds_own_chips(page, selector)):
await input_dispatch.clear(page, selector, timeout=_ACTION_TIMEOUT_MS)
- await input_dispatch.type_keys(
- page, selector, typed, delay=15, timeout=_typing_timeout_ms(typed), secret=secret, replace=True
- )
+ await input_dispatch.type_keys(page, selector, typed, delay=15, timeout=_typing_timeout_ms(typed))
if collateral:
collateral[:] = await _collateral_moved_while_typing(page, collateral)
if not presnapshot_ok or reach != "click":
@@ -16200,7 +16194,7 @@ def build_browser_tools(
held = await box.input_value(timeout=1000)
if not held:
# v1's fallback: a box that drops a programmatic value may still take the key.
- await input_dispatch.type_keys(page, box, char, timeout=1000, secret=secret)
+ await input_dispatch.type_keys(page, box, char, timeout=1000)
held = await box.input_value(timeout=1000)
except Exception:
held = None
@@ -16454,9 +16448,7 @@ def build_browser_tools(
at_end = False
if at_end:
# page.type() would focus the field again, and a fresh focus puts the caret back at the start.
- sent = await _type_keys_before_deadline(
- text, press_end=at_end == "end_key", secret=text != args.get("text", "")
- )
+ sent = await _type_keys_before_deadline(text, press_end=at_end == "end_key")
if sent < len(text):
progress = (
"partway" if text != args.get("text", "") else f"after {sent} of {len(text)} characters"
@@ -16468,9 +16460,7 @@ def build_browser_tools(
error_class="text_not_held",
)
else:
- await input_dispatch.type_keys(
- page, selector, text, timeout=_typing_timeout_ms(text), secret=text != args.get("text", "")
- )
+ await input_dispatch.type_keys(page, selector, text, timeout=_typing_timeout_ms(text))
not_held = await _appended_text_not_held(
field, selector, text, before, text_is_secret=text != args.get("text", "")
)
@@ -16519,7 +16509,7 @@ def build_browser_tools(
await handle.dispose()
return element
- async def _type_keys_before_deadline(text: str, *, press_end: bool, secret: bool) -> int:
+ async def _type_keys_before_deadline(text: str, *, press_end: bool) -> int:
# One key at a time: a single keyboard.type(text) keeps typing in the browser after its caller stops
# waiting, into whatever field the next action focuses. The keyboard is the page's; a frame has none.
top = _current_page()
@@ -16534,7 +16524,7 @@ def build_browser_tools(
if remaining <= 0:
return sent
try:
- await asyncio.wait_for(input_dispatch.type_keys(top, None, key, secret=secret), remaining)
+ await asyncio.wait_for(input_dispatch.type_keys(top, None, key), remaining)
except asyncio.TimeoutError:
return sent
return len(text)
@@ -16629,7 +16619,7 @@ def build_browser_tools(
collateral: list[list[str]] = []
try:
pick, pre_value, pre_own, reach = await _type_and_commit(
- page, selector, text, rounds=3, focus_fallback=True, collateral=collateral, secret=text_is_secret
+ page, selector, text, rounds=3, focus_fallback=True, collateral=collateral
)
except _FieldCovered as exc:
return _covered_error(exc.selector, exc.occluder)
@@ -16888,7 +16878,6 @@ def build_browser_tools(
pre_anchor_closed = None
async def _open_and_enumerate() -> tuple[dict[str, Any] | None, ToolResult | None]:
- await input_dispatch.approach(page, selector)
try:
await page.evaluate(_PRESNAPSHOT_JS)
except Exception:
@@ -17951,9 +17940,7 @@ def build_browser_tools(
# PREVIOUS query left on screen as pre-existing, and a widget that keeps its row
# nodes across a re-search would then have no reaction to show at all.
await page.evaluate(_PRESNAPSHOT_JS)
- await input_dispatch.type_keys(
- page, selector, rung, delay=15, timeout=_typing_timeout_ms(rung), replace=True
- )
+ await input_dispatch.type_keys(page, selector, rung, delay=15, timeout=_typing_timeout_ms(rung))
except Exception:
return []
rung_pick = await _commit_typeahead(
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.py
index 2a89a4bac..1135be0ac 100644
--- a/tests/unit/copilot_test_helpers.py
+++ b/tests/unit/copilot_test_helpers.py
@@ -270,7 +270,11 @@ def install_get_run_results_harness(
"""Stub the collaborators ``_get_run_results`` reaches and return the ctx to call it with; the run pool is
``wr-1`` plus ``other_runs``, and run lookup and history listing honor their arguments."""
pool = [harness_run("wr-1", status=run_status), *(other_runs or [])]
- workflow = SimpleNamespace(workflow_definition=SimpleNamespace(parameters=workflow_parameters or []))
+ workflow = SimpleNamespace(
+ created_by=None,
+ modified_at=HARNESS_RUN_CREATED_AT,
+ workflow_definition=SimpleNamespace(parameters=workflow_parameters or [], blocks=[]),
+ )
async def get_workflow_run(workflow_run_id: str, organization_id: str | None = None) -> SimpleNamespace | None:
return next(
diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.py
index d090234df..6b5e5cf51 100644
--- a/tests/unit/test_browser_effect_sink_enumeration.py
+++ b/tests/unit/test_browser_effect_sink_enumeration.py
@@ -75,15 +75,13 @@ _DISCOVERED_BROWSER_API_CALLS = {
"skyvern/forge/sdk/event/factory.py": Counter({"click": 1, "wheel": 1}),
"skyvern/forge/taskv3/input_dispatch.py": Counter(
{
- "click": 6,
- "evaluate": 2,
+ "click": 4,
+ "evaluate": 1,
"fill": 3,
"focus": 2,
"hover": 1,
- "insert_text": 1,
"press": 3,
"press_sequentially": 1,
- "scroll_into_view_if_needed": 1,
"select_option": 2,
"set_files": 1,
"set_input_files": 1,
@@ -144,8 +142,8 @@ _EVALUATE_CALLERS = {
}
),
"skyvern/webeye/real_browser_state.py": Counter({"stop_page_loading": 1}),
- # The settle wait after a treatment cursor approach, and the hidden-native-control click V3 fires in-page.
- "skyvern/forge/taskv3/input_dispatch.py": Counter({"approach": 1, "js_click": 1}),
+ # The hidden-native-control click V3 fires in-page.
+ "skyvern/forge/taskv3/input_dispatch.py": Counter({"js_click": 1}),
# OTP box/scope/document marker writes, visual masks and read-only hit-test geometry.
"skyvern/webeye/utils/dom.py": Counter(
{"apply_secret_visual_mask": 1, "mark_totp_box": 2, "blur": 1, "_pointer_interceptor_matches_label": 1}
@@ -261,21 +259,21 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:
}
assert observed == _DISCOVERED_BROWSER_API_CALLS
- assert sum(sum(methods.values()) for methods in observed.values()) == 211
+ assert sum(sum(methods.values()) for methods in observed.values()) == 206
handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)
classified_non_browser = Counter(
{signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES}
)
assert classified_non_browser == _NON_BROWSER_CANDIDATES
assert sum(_NON_BROWSER_CANDIDATES.values()) == 7
- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 204
+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 199
def test_every_raw_evaluate_call_is_classified() -> None:
observed = {path: callers for path in _owned_source_paths() if (callers := _callers_for_method(path, "evaluate"))}
assert observed == _EVALUATE_CALLERS
- assert sum(sum(callers.values()) for callers in observed.values()) == 39
+ assert sum(sum(callers.values()) for callers in observed.values()) == 38
def test_every_cdp_dispatch_is_classified_by_exact_command() -> None:
@@ -284,7 +282,7 @@ def test_every_cdp_dispatch_is_classified_by_exact_command() -> None:
assert observed == _CDP_SENDS
-# Task V3 sends input only through input_dispatch, which picks the plain or the humanized transport per run.
+# Task V3 sends input only through input_dispatch.
_INPUT_DISPATCH = Path("skyvern/forge/taskv3/input_dispatch.py")
_TASKV3_BANNED_INPUT_METHODS = frozenset(
"""check click dblclick dispatch_event drag_and_drop drag_to fill focus hover insert_text press press_sequentially select_option
@@ -295,23 +293,6 @@ _IN_PAGE_INPUT_JS = re.compile(r"\.click\(\)|dispatchEvent\(|new (Mouse|Keyboard
# `clear` is also a list and dict method, so it is banned only on a receiver shaped like an element.
_ELEMENT_SOURCES = frozenset({"locator", "query_selector", "element_handle", "nth", "first", "last"})
-# Where each input_dispatch gesture reaches the event strategy factory: a gesture that loses one sends treatment
-# runs down the plain path without anything else going red.
-_INPUT_DISPATCH_FACTORY_CALLS = Counter(
- {
- ("_move_to_element", "move_cursor"): 1,
- ("approach", "move_to_element"): 1,
- ("click", "click_element"): 1,
- ("click", "move_to_element"): 1,
- ("click_at", "move_cursor"): 1,
- ("hover", "move_to_element"): 1,
- ("clear", "clear_field"): 1,
- ("type_keys", "type_text"): 1,
- ("wheel", "scroll_by"): 1,
- ("arm_fields", "registered_profile"): 1,
- }
-)
-
def _enclosing_function(parents: dict[ast.AST, ast.AST], node: ast.AST) -> str:
current = node
@@ -385,18 +366,3 @@ def test_taskv3_input_flows_only_through_input_dispatch() -> None:
sites += _direct_input_sites(Path("skyvern/webeye/utils/captcha_solver.py"), "_solve_challenge_ladder_impl")
assert sites == []
-
-
-def test_input_dispatch_reaches_the_event_strategy_factory_from_every_humanized_gesture() -> None:
- tree = ast.parse(_INPUT_DISPATCH.read_text())
- parents = {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)}
- observed = Counter(
- (_enclosing_function(parents, node), node.func.attr)
- for node in ast.walk(tree)
- if isinstance(node, ast.Call)
- and isinstance(node.func, ast.Attribute)
- and isinstance(node.func.value, ast.Name)
- and node.func.value.id == "EventStrategyFactory"
- )
-
- assert observed == _INPUT_DISPATCH_FACTORY_CALLS
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 9fb5fd8e0..d2b57225e 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -6,9 +6,9 @@ import hashlib
import json
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from types import SimpleNamespace
-from typing import Any
+from typing import Any, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestConnectFailure,
BuildTestEvidencePacket,
BuildTestFailedOperation,
+ BuildTestPacketAiFallbackBlock,
BuildTestPacketDownload,
BuildTestPacketFailure,
BuildTestPacketLocatorObservation,
@@ -44,6 +45,9 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestPacketRegisteredOutput,
BuildTestPacketRequestedOutput,
BuildTestPacketUnfinishedItem,
+ LoopInputFact,
+ LoopInputs,
+ LoopSelectedInput,
PostRunPagePathFailure,
RecordedBuildTestOutcome,
authored_block_signatures_from_workflow,
@@ -65,6 +69,7 @@ from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, Copilo
from skyvern.forge.sdk.copilot.enforcement import _summarize_tool_output
from skyvern.forge.sdk.copilot.failure_tracking import selector_identity_from_failure
from skyvern.forge.sdk.copilot.output_utils import (
+ _BUILD_TEST_PACKET_MAX_CHARS,
_INTERNAL_RUN_OUTCOME_RECORDED_KEY,
_compact_packet_for_aggregate_limit,
_compacted_newest_labelled,
@@ -92,16 +97,32 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (
_run_blocks_and_collect_debug,
_verify_and_record_run_blocks_result,
build_test_evidence_packet,
+ finalize_build_test_result,
+ loop_definition_facts,
+ loop_selected_input,
+ project_run_results_page,
+ with_loop_observations,
)
from skyvern.forge.sdk.copilot.workflow_yaml import runner_code_block_associations
from skyvern.forge.sdk.db.exceptions import CopilotProposalConflictError
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome, UnresolvedRuntimeFailure
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
+from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
-from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.forge.sdk.workflow.models.block import CodeBlock, ForLoopBlock
from skyvern.forge.sdk.workflow.models.google_sheets_blocks import GoogleSheetsWriteBlock
-from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
-from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition, WorkflowRunStatus
+from skyvern.forge.sdk.workflow.models.parameter import (
+ OutputParameter,
+ ParameterType,
+ WorkflowParameter,
+ WorkflowParameterType,
+)
+from skyvern.forge.sdk.workflow.models.workflow import (
+ COPILOT_TEST_WORKFLOW_CREATOR,
+ Workflow,
+ WorkflowDefinition,
+ WorkflowRunStatus,
+)
from skyvern.schemas.self_heal import HealEpisode, HealSkipReason, HealStatus
from skyvern.schemas.workflows import BlockType
from skyvern.services import workflow_service as workflow_service_module
@@ -110,10 +131,12 @@ from skyvern.webeye.actions.actions import ActionStatus
from skyvern.webeye.browser_artifacts import BrowserArtifacts
from tests.unit.copilot_test_helpers import (
HANDBACK_WORKFLOW_YAML,
+ HARNESS_RUN_CREATED_AT,
SEARCH_THEN_SELECT_WORKFLOW_YAML,
count_record_and_send,
failed_second_factor_run,
handback_ctx,
+ inert_approval_workflow,
install_get_run_results_harness,
install_run_blocks_harness,
make_copilot_ctx,
@@ -2038,6 +2061,8 @@ async def test_failed_run_complete_fact_packet_reaches_ordinary_repair_input(
)
run_workflow = SimpleNamespace(
workflow_id="wf_run_snapshot",
+ created_by=None,
+ modified_at=datetime(2026, 4, 21, 12, 0),
organization_id=ctx.organization_id,
workflow_definition=SimpleNamespace(
parameters=[output_parameter],
@@ -8378,3 +8403,549 @@ async def test_a_run_without_an_ai_fallback_reads_exactly_as_before(monkeypatch:
assert reads[0] == reads[1]
for absent in ("ai_fallback", "parent_block_label"):
assert absent not in reads[0]
+
+
+_PARSE_THEN_LOOP_YAML = """
+title: parse then loop
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: file_url_parser
+ label: parse_rows
+ file_url: https://fixture.test/rows.csv
+ file_type: csv
+ next_block_label: visit_each_row
+ - block_type: for_loop
+ label: visit_each_row
+ loop_over_parameter_key: parse_rows_output
+ {reference}
+ loop_blocks:
+ - block_type: goto_url
+ label: open_row_url
+ url: "{{{{ current_value.url }}}}"
+"""
+
+_LoopFactRow = tuple[
+ str,
+ str | None,
+ str | None,
+ str | None,
+ LoopSelectedInput,
+ int | None,
+ list[int | Literal["not_recorded"]] | None,
+ int | None,
+]
+
+_NESTED_LOOP_YAML = """
+title: nested loop
+workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ key: groups
+ workflow_parameter_type: json
+ default_value: [{"urls": ["https://fixture.test/a"]}]
+ blocks:
+ - block_type: for_loop
+ label: each_group
+ loop_over_parameter_key: groups
+ loop_blocks:
+ - block_type: for_loop
+ label: each_url
+ loop_variable_reference: current_value.urls
+ loop_blocks:
+ - block_type: goto_url
+ label: open_url
+ url: "{{ current_value }}"
+"""
+
+_REFERENCE_NAMES_A_PARAMETER_YAML = """
+title: reference names a parameter
+workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ key: groups
+ workflow_parameter_type: json
+ default_value: [{}]
+ - parameter_type: workflow
+ key: other_groups
+ workflow_parameter_type: json
+ default_value: [{}]
+ blocks:
+ - block_type: for_loop
+ label: each_group
+ loop_over_parameter_key: other_groups
+ loop_variable_reference: "{{ groups }}"
+ loop_blocks:
+ - block_type: goto_url
+ label: open_url
+ url: https://fixture.test/a
+"""
+
+
+def _parse_then_loop_yaml(reference: str | None) -> str:
+ return _PARSE_THEN_LOOP_YAML.format(
+ reference=f"loop_variable_reference: {json.dumps(reference)}" if reference else ""
+ )
+
+
+async def test_failed_both_inputs_run_reports_the_dispatched_versions_selected_reference_to_the_model(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ parser_row = terminal_extraction_block("completed", label="parse_rows").model_copy(
+ update={"block_type": BlockType.FILE_URL_PARSER}
+ )
+ failed_loop_row = terminal_extraction_block(
+ "failed", label="visit_each_row", failure_reason="Failed to get loop values."
+ ).model_copy(update={"block_type": BlockType.FOR_LOOP})
+ harness = await install_run_blocks_harness(
+ monkeypatch,
+ workflow_yaml=_parse_then_loop_yaml("current_value"),
+ polled_status="failed",
+ dispatch_to_worker=True,
+ terminal_blocks=[parser_row, failed_loop_row],
+ )
+ dispatched = harness["workflow"].model_copy(deep=True, update={"workflow_id": "w_dispatch_v8", "version": 8})
+ forge_app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version = AsyncMock(return_value=dispatched)
+
+ async def execute_and_rewrite_reference(**_kwargs: object) -> None:
+ loop = dispatched.workflow_definition.blocks[1]
+ assert isinstance(loop, ForLoopBlock)
+ loop.loop_variable_reference = "extracted_loop_values_runtime"
+
+ harness["worker_execute"].side_effect = execute_and_rewrite_reference
+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")
+ ctx.frontier_resume_session_id = "pbs_run"
+
+ result = await _run_blocks_and_collect_debug(
+ {"block_labels": ["parse_rows", "visit_each_row"], "parameters": {}}, ctx
+ )
+ finalize_build_test_result(ctx, source_tool="update_and_run_blocks", result=result)
+ model_facing = sanitize_tool_result_for_llm("update_and_run_blocks", result)["data"]
+
+ parser_output_id = dispatched.workflow_definition.blocks[0].output_parameter.output_parameter_id
+ assert "loop_inputs" not in model_facing
+ assert model_facing["build_test_packet"]["run"]["loop_inputs"] == {
+ "workflow_run_id": "wr_paused",
+ "workflow_id": "w_dispatch_v8",
+ "workflow_permanent_id": "wfp-1",
+ "version": 8,
+ "loops": [
+ {
+ "block_label": "visit_each_row",
+ "loop_over_parameter_key": "parse_rows_output",
+ "producer_block_label": "parse_rows",
+ "producer_output_parameter_id": parser_output_id,
+ "loop_variable_reference": "current_value",
+ "selected_input": "loop_variable_reference",
+ "run_rows": 1,
+ "loop_values_counts": ["not_recorded"],
+ }
+ ],
+ }
+
+
+ ("workflow_yaml", "rows", "expected"),
+ [
+ pytest.param(
+ _parse_then_loop_yaml(None),
+ [
+ terminal_extraction_block("completed", label="visit_each_row").model_copy(
+ update={"block_type": BlockType.FOR_LOOP, "loop_values": ["a", "b"]}
+ )
+ ],
+ [("visit_each_row", None, "parse_rows_output", None, "loop_over_parameter_key", 1, [2], None)],
+ id="typed-only",
+ ),
+ pytest.param(
+ _parse_then_loop_yaml("{{ parse_rows_output.rows }}"),
+ [],
+ [
+ (
+ "visit_each_row",
+ None,
+ "parse_rows_output",
+ "{{ parse_rows_output.rows }}",
+ "loop_variable_reference",
+ None,
+ None,
+ None,
+ )
+ ],
+ id="explicit-expression-never-reached",
+ ),
+ pytest.param(
+ _REFERENCE_NAMES_A_PARAMETER_YAML,
+ [
+ terminal_extraction_block("completed", label="each_group").model_copy(
+ update={"block_type": BlockType.FOR_LOOP, "loop_values": [{}]}
+ )
+ ],
+ [("each_group", None, None, "{{ groups }}", "loop_variable_reference", 1, [1], None)],
+ id="reference-names-a-parameter",
+ ),
+ pytest.param(
+ _NESTED_LOOP_YAML,
+ [
+ terminal_extraction_block("completed", label="each_group").model_copy(
+ update={"block_type": BlockType.FOR_LOOP, "loop_values": [{}]}
+ )
+ ]
+ + [
+ terminal_extraction_block("completed", label="each_url").model_copy(
+ update={"block_type": BlockType.FOR_LOOP, "loop_values": list(range(minute)) or None}
+ )
+ for minute in range(12)
+ ],
+ [
+ ("each_group", None, "groups", None, "loop_over_parameter_key", 1, [1], None),
+ (
+ "each_url",
+ "each_group",
+ None,
+ "current_value.urls",
+ "loop_variable_reference",
+ 12,
+ list(range(2, 12)),
+ 2,
+ ),
+ ],
+ id="nested-keeps-newest-ten-rows",
+ ),
+ ],
+)
+async def test_loop_input_facts_report_each_loops_own_declared_and_recorded_input(
+ monkeypatch: pytest.MonkeyPatch,
+ workflow_yaml: str,
+ rows: list[WorkflowRunBlock],
+ expected: list[_LoopFactRow],
+) -> None:
+ monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
+ definitions = loop_definition_facts(await inert_approval_workflow(workflow_yaml, workflow_id="w_source"))
+ assert definitions is not None
+ facts = with_loop_observations(definitions, "wr_paused", rows)
+
+ assert [
+ (
+ loop.block_label,
+ loop.enclosing_loop_label,
+ loop.loop_over_parameter_key,
+ loop.loop_variable_reference,
+ loop.selected_input,
+ loop.run_rows,
+ loop.loop_values_counts,
+ loop.loop_values_counts_omitted,
+ )
+ for loop in facts.loops
+ ] == expected
+
+
+async def test_a_loop_over_id_from_an_older_version_still_names_its_producer_by_key(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
+ workflow = await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")
+ parser, loop = workflow.workflow_definition.blocks
+ assert isinstance(loop, ForLoopBlock) and isinstance(loop.loop_over, OutputParameter)
+ loop.loop_over = loop.loop_over.model_copy(update={"output_parameter_id": "op_from_version_2"})
+
+ definitions = loop_definition_facts(workflow)
+
+ assert definitions is not None
+ [fact] = definitions.loops
+ assert (fact.producer_block_label, fact.producer_output_parameter_id) == (
+ "parse_rows",
+ parser.output_parameter.output_parameter_id,
+ )
+
+
+ ("reference", "loop_over", "expected"),
+ [
+ ("current_value", True, "loop_variable_reference"),
+ ("", True, "loop_over_parameter_key"),
+ (None, False, "none"),
+ ],
+)
+async def test_selected_input_matches_the_branch_the_real_for_loop_getter_takes(
+ reference: str | None, loop_over: bool, expected: str
+) -> None:
+ now = datetime(2026, 4, 21, 12, 0, tzinfo=UTC)
+ block = ForLoopBlock(
+ label="visit_each_row",
+ output_parameter=OutputParameter(
+ output_parameter_id="op_loop",
+ key="visit_each_row_output",
+ workflow_id="w",
+ created_at=now,
+ modified_at=now,
+ ),
+ loop_blocks=[],
+ loop_over=(
+ WorkflowParameter(
+ workflow_parameter_id="wp_rows",
+ workflow_parameter_type=WorkflowParameterType.JSON,
+ key="rows",
+ workflow_id="w",
+ created_at=now,
+ modified_at=now,
+ )
+ if loop_over
+ else None
+ ),
+ loop_variable_reference=reference,
+ complete_if_empty=True,
+ )
+ branch_values = {"loop_variable_reference": ["from_reference"], "loop_over_parameter_key": ["from_key"], "none": []}
+ context = MagicMock()
+ context.get_value.return_value = branch_values["loop_over_parameter_key"]
+
+ with patch.object(
+ ForLoopBlock,
+ "get_values_from_loop_variable_reference",
+ AsyncMock(return_value=branch_values["loop_variable_reference"]),
+ ):
+ observed = await block.get_loop_over_parameter_values(context, "wr", "wrb")
+
+ assert loop_selected_input(block) == expected
+ assert observed == branch_values[expected]
+
+
+async def test_read_run_reports_the_executed_versions_loop_inputs_not_the_current_draft(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
+ executed = (
+ await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")
+ ).model_copy(update={"modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)})
+ ctx = install_get_run_results_harness(
+ monkeypatch,
+ blocks=[
+ terminal_extraction_block(
+ "failed", label="visit_each_row", failure_reason="Failed to get loop values."
+ ).model_copy(update={"block_type": BlockType.FOR_LOOP})
+ ],
+ )
+ run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=executed)
+ ctx.staged_workflow = await inert_approval_workflow(_parse_then_loop_yaml(None), workflow_id="w_source")
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+ page = sanitize_tool_result_for_llm("get_run_results", project_run_results_page(result, {}))["data"]
+
+ assert page["loop_inputs"]["workflow_run_id"] == "wr-1"
+ assert page["loop_inputs"]["workflow_id"] == "w_source"
+ [loop] = page["loop_inputs"]["loops"]
+ assert (loop["loop_variable_reference"], loop["selected_input"], loop["loop_values_counts"]) == (
+ "current_value",
+ "loop_variable_reference",
+ ["not_recorded"],
+ )
+ assert "Failed to get loop values" not in json.dumps(page["loop_inputs"])
+
+ run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=None)
+ unresolved = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+ assert unresolved["data"]["loop_inputs"] == {"executed_definition": "unavailable"}
+ packet = build_test_evidence_packet(_locator_packet_ctx(), unresolved)
+ assert packet.run.loop_inputs is not None
+ assert packet.run.loop_inputs.model_dump() == {"executed_definition": "unavailable"}
+
+
+ ("workflow_yaml", "created_by", "expected_unavailable"),
+ [
+ (_parse_then_loop_yaml("current_value"), None, True),
+ (_parse_then_loop_yaml("current_value"), COPILOT_TEST_WORKFLOW_CREATOR, False),
+ (SEARCH_THEN_SELECT_WORKFLOW_YAML, None, True),
+ ],
+ ids=[
+ "saved-version-overwritten-after-run",
+ "test-version-soft-deleted-after-run",
+ "loop-removed-from-saved-version-after-run",
+ ],
+)
+async def test_read_run_withholds_loop_inputs_when_the_workflow_row_changed_after_the_run(
+ monkeypatch: pytest.MonkeyPatch, workflow_yaml: str, created_by: str | None, expected_unavailable: bool
+) -> None:
+ monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
+ executed = (await inert_approval_workflow(workflow_yaml, workflow_id="w_source")).model_copy(
+ update={"created_by": created_by, "modified_at": HARNESS_RUN_CREATED_AT + timedelta(minutes=5)}
+ )
+ ctx = install_get_run_results_harness(
+ monkeypatch,
+ blocks=[
+ terminal_extraction_block(
+ "failed", label="visit_each_row", failure_reason="Failed to get loop values."
+ ).model_copy(update={"block_type": BlockType.FOR_LOOP})
+ ],
+ )
+ run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=executed)
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+
+ assert (result["data"]["loop_inputs"] == {"executed_definition": "unavailable"}) is expected_unavailable
+
+
[email protected]("fault", ["aware_modified_at", "loop_definition_facts", "with_loop_observations"])
+async def test_read_run_degrades_loop_inputs_to_unavailable_instead_of_failing(
+ monkeypatch: pytest.MonkeyPatch, fault: str
+) -> None:
+ monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
+ modified_at = (
+ HARNESS_RUN_CREATED_AT.replace(tzinfo=UTC) + timedelta(minutes=5)
+ if fault == "aware_modified_at"
+ else HARNESS_RUN_CREATED_AT - timedelta(minutes=1)
+ )
+ executed = (
+ await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")
+ ).model_copy(update={"modified_at": modified_at})
+ ctx = install_get_run_results_harness(
+ monkeypatch,
+ blocks=[
+ terminal_extraction_block("failed", label="visit_each_row").model_copy(
+ update={"block_type": BlockType.FOR_LOOP}
+ )
+ ],
+ )
+ run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=executed)
+ if fault != "aware_modified_at":
+ monkeypatch.setattr(run_execution_module, fault, MagicMock(side_effect=ValueError("projection failed")))
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+
+ assert result["data"]["loop_inputs"] == {"executed_definition": "unavailable"}
+
+
[email protected]("fault", ["loop_definition_facts", "with_loop_observations"])
+async def test_build_test_loop_inputs_degrade_to_unavailable_when_the_projection_raises(
+ monkeypatch: pytest.MonkeyPatch, fault: str
+) -> None:
+ monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
+ workflow = await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")
+ monkeypatch.setattr(run_execution_module, fault, MagicMock(side_effect=ValueError("projection failed")))
+ execution = run_execution_module._RunExecution(
+ snapshot=run_execution_module.CopilotExecutionSnapshot(
+ provenance="staged",
+ workflow=workflow,
+ workflow_parameters=(),
+ output_parameters=(),
+ workflow_yaml="",
+ ),
+ workflow_yaml="",
+ metadata={},
+ associations={},
+ source_at_start=None,
+ unbound_keys=[],
+ explicit_blank=False,
+ loop_definitions=run_execution_module._loop_definitions_or_unavailable(workflow),
+ )
+ data: dict[str, Any] = {}
+
+ run_execution_module._attach_loop_inputs(data, execution, "wr-1", [])
+
+ assert data["loop_inputs"] == {"executed_definition": "unavailable"}
+
+
+async def test_read_run_reports_no_loop_inputs_for_an_unresolved_workflow_that_ran_no_loop(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = install_get_run_results_harness(
+ monkeypatch, blocks=[terminal_extraction_block("failed", label="open_result")]
+ )
+ run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=None)
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+
+ assert "loop_inputs" not in result["data"]
+ assert build_test_evidence_packet(_locator_packet_ctx(), result).run.loop_inputs is None
+
+
+async def test_prior_run_packet_and_handoff_carry_the_notice_for_a_truncated_loop_reference(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
+ executed = (
+ await inert_approval_workflow(_parse_then_loop_yaml("current_value." + "a" * 400), workflow_id="w_source")
+ ).model_copy(update={"modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)})
+ ctx = install_get_run_results_harness(
+ monkeypatch,
+ blocks=[
+ terminal_extraction_block("failed", label="visit_each_row").model_copy(
+ update={"block_type": BlockType.FOR_LOOP}
+ )
+ ],
+ )
+ run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run = AsyncMock(return_value=executed)
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+ packet = build_test_evidence_packet(_locator_packet_ctx(), result)
+
+ assert isinstance(packet.run.loop_inputs, LoopInputs)
+ assert len(packet.run.loop_inputs.loops[0].loop_variable_reference or "") <= 200
+ assert any("loop_variable_reference" in notice for notice in packet.omission_notices)
+ handoff = project_direct_test_handoff_packet_for_llm(packet)
+ assert any("loop_variable_reference" in notice for notice in handoff.omission_notices)
+
+
[email protected]("ai_fallback_rows", [0, 12])
+def test_many_long_loop_facts_stay_inside_the_packet_budget(ai_fallback_rows: int) -> None:
+ long_label = "l" * 400
+ loop_inputs = LoopInputs(
+ workflow_run_id="wr_1",
+ workflow_id="w_dispatch",
+ workflow_permanent_id="wpid",
+ version=3,
+ loops=[
+ LoopInputFact(
+ block_label=f"{long_label}{index}",
+ enclosing_loop_label=long_label,
+ loop_over_parameter_key=f"{long_label}_output",
+ producer_block_label=long_label,
+ loop_variable_reference="r" * 5_000,
+ selected_input="loop_variable_reference",
+ run_rows=12 + index,
+ loop_values_counts=[1_000_000] * 10,
+ loop_values_counts_omitted=2,
+ )
+ for index in range(20)
+ ],
+ )
+ packet = _oversized_packet(build_test_evidence_packet(_locator_packet_ctx(), _failed_run_result(None)))
+ ai_fallback_blocks = [
+ BuildTestPacketAiFallbackBlock(
+ block_label=f"{long_label}{index}",
+ status="fired_failed",
+ task_id=f"tsk_{index}",
+ failing_line=index,
+ failure_text="f" * 2_000,
+ recovery_failure_text="g" * 2_000,
+ )
+ for index in range(ai_fallback_rows)
+ ]
+ packet = packet.model_copy(
+ update={
+ "run": packet.run.model_copy(update={"loop_inputs": loop_inputs}),
+ "ai_fallback_blocks": ai_fallback_blocks or None,
+ }
+ )
+
+ projected = project_build_test_packet_for_llm(packet)
+
+ serialized = json.dumps(projected.model_dump(mode="json", exclude_none=True), ensure_ascii=False)
+ assert len(serialized) <= _BUILD_TEST_PACKET_MAX_CHARS
+ assert isinstance(projected.run.loop_inputs, LoopInputs)
+ # Loop facts give way before canonical_workflow_yaml is shortened.
+ assert [loop.run_rows for loop in projected.run.loop_inputs.loops] == [12 + index for index in range(6)]
+ assert projected.run.loop_inputs.loops_omitted == 14
+ assert all(len(loop.loop_variable_reference or "") <= 200 for loop in projected.run.loop_inputs.loops)
+ assert all(loop.selected_input == "loop_variable_reference" for loop in projected.run.loop_inputs.loops)
+ assert all(loop.loop_values_counts == [1_000_000] * 10 for loop in projected.run.loop_inputs.loops)
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 5b8a00bd3..b608f04c0 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -3475,6 +3475,8 @@ async def test_completed_missing_output_complete_fact_packet_reaches_ordinary_re
)
run_workflow = SimpleNamespace(
organization_id=ctx.organization_id,
+ created_by=None,
+ modified_at=datetime(2026, 4, 21, 12, 0),
workflow_definition=SimpleNamespace(
parameters=[output_parameter],
blocks=[SimpleNamespace(label="open_result", block_type="CODE", output_parameter=output_parameter)],
diff --git a/tests/unit/test_copilot_locator_inspection.py b/tests/unit/test_copilot_locator_inspection.py
index 01205509a..eae56846e 100644
--- a/tests/unit/test_copilot_locator_inspection.py
+++ b/tests/unit/test_copilot_locator_inspection.py
@@ -921,15 +921,17 @@ async def test_prior_run_result_uses_its_exact_failed_row_and_workflow_for_typed
modified_at=now,
)
workflow = SimpleNamespace(
- workflow_definition={
- "blocks": [
+ created_by=None,
+ modified_at=datetime(2026, 4, 21, 12, 0),
+ workflow_definition=SimpleNamespace(
+ blocks=[
{
"label": "click_submit",
"block_type": "code",
"code": 'await page.locator("#submit").click()',
}
]
- }
+ ),
)
database = SimpleNamespace(
workflow_runs=SimpleNamespace(get_workflow_run=AsyncMock(return_value=run)),
diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.py
index 6be86dd0c..11f7245d5 100644
--- a/tests/unit/test_copilot_mcp_adapter.py
+++ b/tests/unit/test_copilot_mcp_adapter.py
@@ -1,4 +1,5 @@
import asyncio
+import copy
import json
from collections.abc import AsyncIterator, Callable, Iterator
from contextlib import asynccontextmanager
@@ -9,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock
from urllib.parse import quote, urlparse
import pytest
+import yaml
from agents.mcp.util import MCPUtil
from agents.tool_context import ToolContext
from fastmcp import FastMCP
@@ -25,7 +27,7 @@ from skyvern.forge.sdk.copilot import agent as copilot_agent
from skyvern.forge.sdk.copilot import mcp_adapter
from skyvern.forge.sdk.copilot import runtime as copilot_runtime
from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode, resolve_copilot_tool_surface
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
+from skyvern.forge.sdk.copilot.config import AGENT_BLOCKS_ONLY, BlockAuthoringPolicy, CopilotConfig
from skyvern.forge.sdk.copilot.context import USER_FACING_REASON_PARAM
from skyvern.forge.sdk.copilot.enforcement import CopilotTotalTimeoutError
from skyvern.forge.sdk.copilot.mcp_adapter import (
@@ -61,7 +63,14 @@ from skyvern.forge.sdk.copilot.secret_scrub import (
from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS, mcp_hooks
from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
from skyvern.forge.sdk.copilot.tools._shared import _composition_get_structured_evidence_result
-from skyvern.forge.sdk.copilot.tools.mcp_hooks import _build_skyvern_mcp_overlays, get_skyvern_mcp_alias_map
+from skyvern.forge.sdk.copilot.tools.banned_blocks import _block_authoring_violations
+from skyvern.forge.sdk.copilot.tools.mcp_hooks import (
+ _FOR_LOOP_EXAMPLE,
+ _FOR_LOOP_GUIDANCE,
+ _FOR_LOOP_PROPERTY_DESCRIPTIONS,
+ _build_skyvern_mcp_overlays,
+ get_skyvern_mcp_alias_map,
+)
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.webeye.persistent_sessions_manager import (
BrowserOperation,
@@ -3744,3 +3753,56 @@ async def test_actor_reason_actual_mcp_schema_alias_strips_only_metadata() -> No
assert original == {"block_type": "code"}
finally:
await server.cleanup()
+
+
+async def test_for_loop_schema_and_knowledge_state_reference_precedence_without_refusing_both_inputs() -> None:
+ ctx = make_copilot_ctx(api_key="in-process-test-key")
+ aliases = get_skyvern_mcp_alias_map()
+ names = ("get_block_schema", "get_workflow_knowledge")
+ server = SkyvernOverlayMCPServer(
+ transport=mcp,
+ overlays=_build_skyvern_mcp_overlays(),
+ alias_map={name: aliases[name] for name in names},
+ allowlist=frozenset(aliases[name] for name in names),
+ context_provider=lambda: ctx,
+ )
+ await server.connect()
+ try:
+ tools = {tool.name: tool for tool in await server.list_tools()}
+
+ async def call(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+ tool = MCPUtil.to_function_tool(tools[name], server, convert_schemas_to_strict=False)
+ tc = ToolContext(context=ctx, tool_name=name, tool_call_id=f"{name}-call", tool_arguments="{}")
+ output = await tool.on_invoke_tool(tc, json.dumps(arguments))
+ return json.loads(output["text"])["data"]
+
+ schema_data = await call("get_block_schema", {"block_type": "for_loop"})
+ knowledge = (await call("get_workflow_knowledge", {"topics": ["for_loop_block"]}))["sections"]["for_loop_block"]
+ finally:
+ await server.cleanup()
+
+ schema = schema_data["schema"]
+ properties = schema["$defs"][schema["$ref"].removeprefix("#/$defs/")]["properties"]
+ assert {name: properties[name]["description"] for name in _FOR_LOOP_PROPERTY_DESCRIPTIONS} == (
+ _FOR_LOOP_PROPERTY_DESCRIPTIONS
+ )
+ assert schema_data["example"] == _FOR_LOOP_EXAMPLE
+ producer, loop = schema_data["example"]["blocks"]
+ assert loop["loop_over_parameter_key"] == f"{producer['label']}_output"
+ assert "loop_variable_reference" not in loop
+ assert knowledge["content"] == _FOR_LOOP_GUIDANCE
+ assert all(description in _FOR_LOOP_GUIDANCE for description in _FOR_LOOP_PROPERTY_DESCRIPTIONS.values())
+ assert yaml.safe_load(_FOR_LOOP_GUIDANCE.partition("the list):\n")[2]) == _FOR_LOOP_EXAMPLE
+ assert {"loop_over_parameter_key", "loop_variable_reference"} <= set(properties)
+
+ both_inputs = {
+ "block_type": "for_loop",
+ "label": "visit_each_row",
+ "loop_over_parameter_key": "parse_rows_output",
+ "loop_variable_reference": "current_value",
+ "loop_blocks": [{"block_type": "goto_url", "label": "open_row_url", "url": "{{ current_value.url }}"}],
+ }
+ submitted = copy.deepcopy(both_inputs)
+ assert _block_authoring_violations(submitted, AGENT_BLOCKS_ONLY) == []
+ assert submitted == both_inputs
diff --git a/tests/unit/test_copilot_positive_evidence_carriers.py b/tests/unit/test_copilot_positive_evidence_carriers.py
index 0c03a6341..43260a32c 100644
--- a/tests/unit/test_copilot_positive_evidence_carriers.py
+++ b/tests/unit/test_copilot_positive_evidence_carriers.py
@@ -1069,7 +1069,11 @@ async def test_origin_registry_recovers_when_a_later_terminal_read_finds_the_han
created_at=datetime(2026, 4, 21, 12, 0),
trigger_type=None,
)
- workflow = SimpleNamespace(workflow_definition=SimpleNamespace(parameters=[]))
+ workflow = SimpleNamespace(
+ created_by=None,
+ modified_at=datetime(2026, 4, 21, 12, 0),
+ workflow_definition=SimpleNamespace(parameters=[], blocks=[]),
+ )
monkeypatch.setattr(
run_execution_module,
"app",
diff --git a/tests/unit/test_taskv3_input_dispatch.py b/tests/unit/test_taskv3_input_dispatch.py
index 77833f749..8eb848f6b 100644
--- a/tests/unit/test_taskv3_input_dispatch.py
+++ b/tests/unit/test_taskv3_input_dispatch.py
@@ -1,19 +1,13 @@
from __future__ import annotations
-from collections.abc import Awaitable, Callable, Iterator
+from collections.abc import Awaitable, Callable
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
-from playwright.async_api import ElementHandle, Locator, Page
+from playwright.async_api import ElementHandle
-from skyvern.forge import app
-from skyvern.forge.sdk.core import skyvern_context
-from skyvern.forge.sdk.core.skyvern_context import RunArm, SkyvernContext
-from skyvern.forge.sdk.event.base import CursorEventStrategy, InputEventStrategy, ScrollEventStrategy
-from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.taskv3 import input_dispatch
-from skyvern.forge.taskv3.run_arms import HUMANIZED_INPUT_FLAG, resolve_run_arm
Call = tuple[str, tuple[Any, ...], dict[str, Any]]
@@ -33,16 +27,6 @@ class _Device:
class _FakeLocator(_Device):
def __init__(self, page: _FakePage, selector: str) -> None:
super().__init__(page.calls, f"locator({selector})")
- self.page = page
- self.first = self
-
-
-class _FakeHandle(_Device):
- def __init__(self, page: _FakePage) -> None:
- super().__init__(page.calls, "handle")
-
- async def bounding_box(self) -> dict[str, float]:
- return {"x": 10.0, "y": 20.0, "width": 4.0, "height": 6.0}
class _FakePage(_Device):
@@ -52,168 +36,18 @@ class _FakePage(_Device):
self.keyboard = _Device(self.calls, "keyboard")
self.mouse = _Device(self.calls, "mouse")
- def __getattr__(self, attr: str) -> Callable[..., Awaitable[None]]:
- # A top-level page has neither, which is how the dispatcher tells it from a frame.
- if attr in ("page", "parent_frame"):
- raise AttributeError(attr)
- return super().__getattr__(attr)
-
def locator(self, selector: str) -> _FakeLocator:
return _FakeLocator(self, selector)
-class _RecordingCursor(CursorEventStrategy):
- def __init__(self, log: list[Call]) -> None:
- self._log = log
-
- async def move_to(self, page: Page, x: float, y: float) -> None:
- self._log.append(("cursor.move_to", (x, y), {}))
-
- async def move_to_element(self, page: Page, locator: Locator) -> tuple[float, float]:
- self._log.append(("cursor.move_to_element", (locator,), {}))
- return 0.0, 0.0
-
- async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:
- self._log.append(("cursor.click", (locator,), {"timeout": timeout}))
-
-
-class _RecordingInput(InputEventStrategy):
- def __init__(self, log: list[Call]) -> None:
- self._log = log
-
- async def type_text(
- self,
- page: Page,
- locator: Locator | None,
- text: str,
- *,
- timeout: float | None,
- delay: float | None = None,
- no_wait_after: bool | None = None,
- allow_batched_playwright: bool = False,
- ) -> None:
- self._log.append(("input.type_text", (locator, text), {"delay": delay}))
-
- async def clear_field(
- self,
- page: Page,
- locator: Locator,
- char_count: int,
- *,
- timeout: float | None,
- force: bool | None = None,
- no_wait_after: bool | None = None,
- ) -> None:
- self._log.append(("input.clear_field", (locator,), {"timeout": timeout}))
-
-
-class _RecordingScroll(ScrollEventStrategy):
- def __init__(self, log: list[Call]) -> None:
- self._log = log
-
- async def scroll_to_element(self, page: Page, locator: Locator) -> None:
- self._log.append(("scroll.scroll_to_element", (locator,), {}))
-
- async def scroll_by(self, page: Page, delta_y: float) -> None:
- self._log.append(("scroll.scroll_by", (delta_y,), {}))
-
-
@pytest.fixture
def page() -> _FakePage:
return _FakePage()
-def humanized_strategies(page: _FakePage) -> Iterator[list[Call]]:
- """A humanized registration, as USE_EVENT_STRATEGIES makes it, logging into the page's own call list."""
- EventStrategyFactory.set_cursor_strategy(_RecordingCursor(page.calls))
- EventStrategyFactory.set_input_strategy(_RecordingInput(page.calls))
- EventStrategyFactory.set_scroll_strategy(_RecordingScroll(page.calls))
- EventStrategyFactory.record_profile("default")
- try:
- yield page.calls
- finally:
- EventStrategyFactory.reset()
-
-
-def _pin(arm: RunArm) -> Iterator[SkyvernContext]:
- context = SkyvernContext()
- context.run_arms = {HUMANIZED_INPUT_FLAG: ("wr_1", arm)}
- skyvern_context.set(context)
- try:
- yield context
- finally:
- skyvern_context.reset()
-
-
-def control() -> Iterator[SkyvernContext]:
- yield from _pin("control")
-
-
-def treatment() -> Iterator[SkyvernContext]:
- yield from _pin("treatment")
-
-
-def _names(calls: list[Call]) -> list[str]:
- return [name for name, _, _ in calls]
-
-
- ("reader", "humanized"),
- [
- (AsyncMock(return_value="treatment"), True),
- (AsyncMock(return_value="control"), False),
- (AsyncMock(return_value=None), False),
- (AsyncMock(return_value="true"), False),
- (AsyncMock(side_effect=RuntimeError("flag service down")), False),
- ],
- ids=["treatment", "control", "no_variant", "unknown_variant", "flag_error"],
-)
-async def test_only_a_treatment_run_sends_its_clicks_through_the_registered_strategy(
- monkeypatch: pytest.MonkeyPatch,
- page: _FakePage,
- humanized_strategies: list[Call],
- reader: AsyncMock,
- humanized: bool,
-) -> None:
- monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)
- context = SkyvernContext()
- skyvern_context.set(context)
- try:
- await resolve_run_arm(context, HUMANIZED_INPUT_FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)
- await input_dispatch.click(page, "#go", timeout=1234)
- finally:
- skyvern_context.reset()
-
- assert _names(page.calls) == (["cursor.click"] if humanized else ["page.click"])
-
-
@pytest.mark.asyncio
-async def test_input_outside_a_resolved_run_is_plain_playwright(
- page: _FakePage, humanized_strategies: list[Call]
-) -> None:
- skyvern_context.reset()
-
- await input_dispatch.click(page, "#go", timeout=1234)
- await input_dispatch.type_keys(page, "#q", "abc", delay=15, timeout=900)
- await input_dispatch.wheel(page, 0, 800)
-
- assert page.calls == [
- ("page.click", ("#go",), {"timeout": 1234}),
- ("page.type", ("#q", "abc"), {"delay": 15, "timeout": 900}),
- ("mouse.wheel", (0, 800), {}),
- ]
-
-
-async def test_control_sends_each_gesture_as_the_exact_playwright_call_it_replaced(
- control: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
-) -> None:
+async def test_each_gesture_is_the_exact_playwright_call_it_replaced(page: _FakePage) -> None:
locator = page.locator("#seg")
- await input_dispatch.approach(page, "#go")
await input_dispatch.click(page, "#go", timeout=5000, force=True)
await input_dispatch.click(page, locator, timeout=2000)
await input_dispatch.click_at(page, 10.0, 20.0)
@@ -252,133 +86,10 @@ async def test_control_sends_each_gesture_as_the_exact_playwright_call_it_replac
@pytest.mark.asyncio
-async def test_control_clicks_an_element_handle_with_only_the_arguments_given(control: SkyvernContext) -> None:
+async def test_an_element_handle_is_clicked_with_only_the_arguments_given() -> None:
handle = MagicMock(spec=ElementHandle)
handle.click = AsyncMock()
await input_dispatch.click_handle(_FakePage(), handle)
handle.click.assert_awaited_once_with()
-
-
-async def test_treatment_routes_pointer_typing_and_scroll_through_the_registered_strategies(
- treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
-) -> None:
- await input_dispatch.click(page, "#go", timeout=1234)
- await input_dispatch.click(page, "#go", timeout=5000, force=True)
- await input_dispatch.click_at(page, 10.0, 20.0)
- await input_dispatch.hover(page, "#menu", timeout=2000)
- await input_dispatch.clear(page, "#f", timeout=100)
- await input_dispatch.wheel(page, 0, 800)
-
- assert _names(page.calls) == [
- "cursor.click",
- # The strategy has no forced click: it moves there, then Playwright clicks exactly as before.
- "cursor.move_to_element",
- "page.click",
- "cursor.move_to",
- "mouse.click",
- "cursor.move_to_element",
- "page.hover",
- "input.clear_field",
- "scroll.scroll_by",
- ]
- assert page.calls[2] == ("page.click", ("#go",), {"timeout": 5000, "force": True})
-
-
-async def test_treatment_clicks_a_locator_through_the_strategy_and_a_handle_after_a_cursor_move(
- treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
-) -> None:
- await input_dispatch.click(page, page.locator("#seg"), timeout=2000)
- await input_dispatch.click_handle(page, _FakeHandle(page), position={"x": 1.0, "y": 2.0}, timeout=1500)
-
- assert _names(page.calls) == ["cursor.click", "cursor.move_to", "handle.click"]
- assert page.calls[1:] == [
- ("cursor.move_to", (12.0, 23.0), {}),
- ("handle.click", (), {"position": {"x": 1.0, "y": 2.0}, "timeout": 1500}),
- ]
-
-
-async def test_treatment_types_through_the_keyboard_after_one_focus(
- treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
-) -> None:
- await input_dispatch.type_keys(page, "#f", "abc", delay=15, timeout=900)
-
- # The strategy gets no locator (given one, a humanized keyboard re-focuses the field before every key) and no
- # delay: as in v1, the registered strategy owns the cadence.
- assert page.calls == [
- ("page.focus", ("#f",), {"timeout": 900}),
- ("input.type_text", (None, "abc"), {"delay": None}),
- ]
-
-
- ("replace", "head_write"),
- [
- (True, ("page.fill", ("#f", "Hello, "), {"timeout": 900})),
- (False, ("keyboard.insert_text", ("Hello, ",), {})),
- ],
- ids=["emptied_field_fills_the_head", "append_inserts_the_head_at_the_caret"],
-)
-async def test_treatment_humanizes_only_the_last_ten_characters_as_v1_does(
- treatment: SkyvernContext,
- page: _FakePage,
- humanized_strategies: list[Call],
- replace: bool,
- head_write: Call,
-) -> None:
- await input_dispatch.type_keys(page, "#f", "Hello, 0123456789", delay=15, timeout=900, replace=replace)
-
- assert page.calls[-2:] == [head_write, ("input.type_text", (None, "0123456789"), {"delay": None})]
-
-
[email protected]("arm", ["control", "treatment"])
-async def test_a_secret_is_typed_key_by_key_exactly_as_before_in_both_arms(
- page: _FakePage, humanized_strategies: list[Call], arm: RunArm
-) -> None:
- # Split one-time-code boxes advance on each keystroke, so neither a fill nor a humanized keyboard may take over.
- for _ in _pin(arm):
- await input_dispatch.type_keys(page, "#code", "s3cret", delay=15, timeout=900, secret=True, replace=True)
- await input_dispatch.type_keys(page, page.locator("#box"), "7", timeout=1000, secret=True)
-
- assert page.calls == [
- ("page.type", ("#code", "s3cret"), {"delay": 15, "timeout": 900}),
- ("locator(#box).press_sequentially", ("7",), {"timeout": 1000}),
- ]
-
-
-async def test_treatment_approach_moves_then_waits_for_the_page_to_settle(
- treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
-) -> None:
- await input_dispatch.approach(page, "#go")
-
- assert _names(page.calls) == ["locator(#go).scroll_into_view_if_needed", "cursor.move_to_element", "page.evaluate"]
-
-
-def test_arm_fields_name_the_arm_whether_it_is_in_effect_and_the_registered_profile(
- treatment: SkyvernContext, humanized_strategies: list[Call]
-) -> None:
- assert input_dispatch.arm_fields() == {
- "humanized_input_arm": "treatment",
- "humanized_input_in_effect": True,
- "global_event_strategy_profile": "default",
- }
-
-
-def test_registered_profile_tells_no_registration_from_an_unnamed_one() -> None:
- EventStrategyFactory.reset()
- try:
- assert EventStrategyFactory.registered_profile() == "none"
- EventStrategyFactory.set_cursor_strategy(_RecordingCursor([]))
- assert EventStrategyFactory.registered_profile() == "unrecorded"
- EventStrategyFactory.record_profile("kernel")
- assert EventStrategyFactory.registered_profile() == "kernel"
- finally:
- EventStrategyFactory.reset()
- assert EventStrategyFactory.registered_profile() == "none"
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 94124220a..66e0c2799 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -45,8 +45,6 @@ from skyvern.exceptions import NO_ADDRESS_RECORD_NAV_ERROR_CODE
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
-from skyvern.forge.sdk.event.default import DefaultInputStrategy
-from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.services import credentials as credentials_module
from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager, WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import _recorded_task_nav_error_codes
@@ -39664,26 +39662,3 @@ async def test_the_failed_read_count_resets_on_a_success_and_on_navigation(monke
await page.goto("http://shop.test/review?again=1")
after = await act(1)
assert after[4] is None and after[1] == 0, after
-
-
-@_skip_no_browser
-async def test_an_appended_credential_never_reaches_the_humanized_keyboard(monkeypatch: pytest.MonkeyPatch) -> None:
- class _RefusingInput(DefaultInputStrategy):
- async def type_text(self, *args: Any, **kwargs: Any) -> None:
- raise AssertionError("a resolved credential was sent through the registered input strategy")
-
- monkeypatch.setattr(settings, "TASK_V3_HUMANIZED_INPUT", True)
- EventStrategyFactory.set_input_strategy(_RefusingInput())
- try:
- async with _content_page('<!doctype html><html><body><input id="t" value="id-"></body></html>') as page:
- tools = build_browser_tools(
- _fixed_page_provider(page),
- resolve_typed_text=lambda text: "real-secret" if text == "placeholder_abc" else text,
- )
- r = await _tool(tools, "type").handler({"selector": "#t", "text": "placeholder_abc", "clear": False})
- held = await page.evaluate("() => document.getElementById('t').value")
- finally:
- EventStrategyFactory.reset()
- assert r.status == "ok", r.content
- assert held == "id-real-secret"