Skip to content

Automate browser based workflows with AI

refactor(SKY-11693): move copilot workflow-YAML processing out of routes into copilot/ (#7337)

Aaron Perez32f005cfa78a
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 9036b0a4a..44f9de55f 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -159,6 +159,10 @@ class CompletionVerificationResult:
# so it must not veto a run whose observable outcome evidence is fully confirmed.
if verdict is not None and _is_definition_plane_abstention(verdict):
continue
+ # An incomplete typed classification contract likewise abstains rather than refutes:
+ # it must not disown a delivered sibling output whose evidence is confirmed.
+ if verdict is not None and _is_incomplete_validation_classification_abstention(verdict):
+ continue
if verdict is not None and verdict.self_emitted_judgment_not_independent:
if _has_independent_satisfied_requested_output_corroborator(verdict_by_id, criterion_id):
satisfied_run_plane_count += 1
@@ -350,6 +354,7 @@ class RunEvidenceSnapshot:
_UNAVAILABLE = CompletionVerificationResult(status="unavailable")
_MISSING_VERDICT_EVIDENCE = "judge did not return a verdict for this criterion"
_INCOMPLETE_VALIDATION_CLASSIFICATION_CONTRACT = "incomplete typed classification contract"
+_INCOMPLETE_VALIDATION_CLASSIFICATION_ABSTENTION_REASON = "validation_classification_incomplete_contract"
_MISSING_REGISTERED_DOWNLOAD_EVIDENCE = "run output did not include a non-empty registered browser download"
_DELIVERED_UNVERIFIED_OUTPUT_SOURCES = frozenset({"runtime_output", "registered_output_parameter"})
@@ -1015,11 +1020,16 @@ def grade_validation_classification_criteria(
output_key = criterion.classification_output_key
expected = criterion.expected_classification
if output_key is None or expected is None:
+ # An incomplete typed contract cannot grade anything, so it fails safe to a non-sinking
+ # abstention like grade_definition_criteria rather than a sinking unsatisfied verdict —
+ # a value-less criterion minted without the fields this arm needs must not disown a
+ # delivered sibling output. It can never satisfy on its own: is_fully_satisfied still
+ # requires a genuinely satisfied run-plane criterion.
verdicts.append(
- _validation_classification_unsatisfied(
- criterion,
- reason_code="no_evidence",
- output_key=output_key or "",
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="unknown",
+ reason_code=_INCOMPLETE_VALIDATION_CLASSIFICATION_ABSTENTION_REASON,
missing_evidence=_INCOMPLETE_VALIDATION_CLASSIFICATION_CONTRACT,
)
)
@@ -2022,6 +2032,10 @@ def _is_definition_plane_abstention(verdict: CriterionVerdict) -> bool:
return verdict.state == "unknown" and verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX)
+def _is_incomplete_validation_classification_abstention(verdict: CriterionVerdict) -> bool:
+ return verdict.state == "unknown" and verdict.reason_code == _INCOMPLETE_VALIDATION_CLASSIFICATION_ABSTENTION_REASON
+
+
def _is_structural_requested_output_abstention(verdict: CriterionVerdict) -> bool:
return verdict.state == "unsatisfied" and verdict.reason_code == _STRUCTURAL_ABSTENTION_REASON_CODE
@@ -2308,6 +2322,8 @@ def only_degraded_blocking(result: CompletionVerificationResult) -> bool:
continue
if verdict is not None and _is_definition_plane_abstention(verdict):
continue
+ if verdict is not None and _is_incomplete_validation_classification_abstention(verdict):
+ continue
if verdict is not None and result.is_structural_contingent_abstention(verdict):
continue
blocking.add(criterion_id)
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index ab8fd8205..36a21c538 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -37,7 +37,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
from skyvern.forge.sdk.copilot.screenshot_utils import enqueue_screenshot_from_result
from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_structure
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
-from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml as _process_workflow_yaml
+from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml as _process_workflow_yaml
from ._shared import _COMPOSITION_STRIPPED_HTML_MAX_CHARS as _COMPOSITION_STRIPPED_HTML_MAX_CHARS
from ._shared import _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS as _CONSECUTIVE_LOOP_GUARD_EXEMPT_TOOLS
diff --git a/skyvern/forge/sdk/copilot/tools/frontier.py b/skyvern/forge/sdk/copilot/tools/frontier.py
index 29cb06cc8..4bcb604f0 100644
--- a/skyvern/forge/sdk/copilot/tools/frontier.py
+++ b/skyvern/forge/sdk/copilot/tools/frontier.py
@@ -30,7 +30,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import (
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
)
-from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
+from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml
from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, get_all_blocks
from skyvern.forge.sdk.workflow.models.parameter import (
RESERVED_PARAMETER_KEYS,
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 50d3f6e82..a47fb18ec 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -133,8 +133,8 @@ from skyvern.forge.sdk.copilot.turn_halt import (
stash_turn_halt_from_blocker_signal,
)
from skyvern.forge.sdk.copilot.typed_value_policy import should_reject_type_text_value
+from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
-from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotRunOutcomeUpdate, WorkflowCopilotStreamMessageType
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.utils.pdf_parser import extract_pdf_file
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index fedaafaa2..b0b59ac61 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -167,7 +167,7 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (
parse_workflow_yaml,
workflow_blocks,
)
-from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
+from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, InsecureCodeDetected
from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.parameter import RESERVED_PARAMETER_KEYS, is_sensitive_workflow_parameter
diff --git a/skyvern/forge/sdk/copilot/workflow_yaml.py b/skyvern/forge/sdk/copilot/workflow_yaml.py
new file mode 100644
index 000000000..873cefa4e
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/workflow_yaml.py
@@ -0,0 +1,446 @@
+"""Copilot workflow-YAML normalization, chain repair, and Workflow conversion."""
+
+from datetime import datetime, timezone
+from typing import Any
+
+import structlog
+import yaml
+
+from skyvern.constants import DEFAULT_LOGIN_PROMPT
+from skyvern.exceptions import WorkflowNotFound
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
+from skyvern.forge.sdk.copilot.code_block_steps import derive_code_block_steps_in_yaml
+from skyvern.forge.sdk.workflow.models.parameter import ParameterType
+from skyvern.forge.sdk.workflow.models.workflow import Workflow
+from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
+from skyvern.schemas.runs import ProxyLocation
+from skyvern.schemas.workflows import (
+ BlockYAML,
+ BranchConditionYAML,
+ ConditionalBlockYAML,
+ ForLoopBlockYAML,
+ LoginBlockYAML,
+ WhileLoopBlockYAML,
+ WorkflowCreateYAMLRequest,
+)
+from skyvern.utils.yaml_loader import safe_load_no_dates
+
+LOG = structlog.get_logger()
+
+
+def _proxy_location_alias_key(value: str) -> str:
+ return "_".join(value.strip().upper().replace("-", "_").split())
+
+
+def _build_copilot_proxy_location_aliases() -> dict[str, str]:
+ aliases: dict[str, str] = {}
+
+ def add(alias: str, proxy_location: ProxyLocation) -> None:
+ aliases[_proxy_location_alias_key(alias)] = proxy_location.value
+
+ for proxy_location in ProxyLocation:
+ add(proxy_location.name, proxy_location)
+ add(proxy_location.value, proxy_location)
+
+ for proxy_location in ProxyLocation.residential_country_locations():
+ add(ProxyLocation.get_country_code(proxy_location), proxy_location)
+
+ add("USA", ProxyLocation.RESIDENTIAL)
+ add("United States", ProxyLocation.RESIDENTIAL)
+ add("United States of America", ProxyLocation.RESIDENTIAL)
+ add("RESIDENTIAL_US", ProxyLocation.RESIDENTIAL)
+ add("UK", ProxyLocation.RESIDENTIAL_GB)
+ add("United Kingdom", ProxyLocation.RESIDENTIAL_GB)
+
+ return aliases
+
+
+_COPILOT_PROXY_LOCATION_ALIASES = _build_copilot_proxy_location_aliases()
+
+
+def _canonicalize_copilot_proxy_location(parsed_yaml: dict[str, Any]) -> None:
+ if "proxy_location" not in parsed_yaml:
+ return
+
+ proxy_location = parsed_yaml.get("proxy_location")
+ if not isinstance(proxy_location, str):
+ return
+
+ canonical = _COPILOT_PROXY_LOCATION_ALIASES.get(_proxy_location_alias_key(proxy_location))
+ if canonical is None:
+ return
+
+ parsed_yaml["proxy_location"] = canonical
+
+
+def _canonicalize_copilot_block_type_aliases(value: Any) -> None:
+ if isinstance(value, dict):
+ block_type = value.get("block_type")
+ if isinstance(block_type, str):
+ value["block_type"] = normalize_copilot_block_type_alias(block_type)
+ for child in value.values():
+ _canonicalize_copilot_block_type_aliases(child)
+ elif isinstance(value, list):
+ for item in value:
+ _canonicalize_copilot_block_type_aliases(item)
+
+
+def _collect_reachable(
+ start_label: str,
+ label_to_block: dict[str, BlockYAML],
+ reachable: set[str],
+) -> None:
+ """Walk the next_block_label chain from start_label, collecting all reachable labels.
+
+ For conditional blocks, also follows branch target chains recursively.
+
+ The ``current not in reachable`` loop guard means the main-chain walk
+ stops early if we hit a node already collected via a branch recursion.
+ This is correct — those downstream nodes and their successors are
+ already in ``reachable`` — but callers should be aware of the coupling.
+ """
+ current: str | None = start_label
+ while current and current in label_to_block and current not in reachable:
+ reachable.add(current)
+ block = label_to_block[current]
+ if isinstance(block, ConditionalBlockYAML):
+ for branch in block.branch_conditions:
+ if branch.next_block_label and branch.next_block_label not in reachable:
+ _collect_reachable(branch.next_block_label, label_to_block, reachable)
+ current = block.next_block_label
+
+
+def _break_cycles(
+ start_label: str,
+ label_to_block: dict[str, BlockYAML],
+) -> bool:
+ """Detect and break circular references in the block chain using DFS.
+
+ Uses a recursion stack to distinguish true back-edges (cycles) from merge
+ points (two branches converging on the same block). When a back-edge is
+ found the offending ``next_block_label`` is set to ``None``, breaking the
+ cycle. Handles both the main chain and conditional branch chains.
+
+ Note: this function operates on a single level of blocks. It does **not**
+ recurse into ``ForLoopBlockYAML.loop_blocks``; nested loops are handled
+ by the recursive ``_repair_next_block_label_chain`` call in Phase 3.
+
+ Returns True if at least one cycle was broken.
+ """
+ visited: set[str] = set()
+ rec_stack: set[str] = set()
+ found_cycle = False
+
+ def _follow_edge(target: str | None, edge_owner: BlockYAML | BranchConditionYAML, parent_label: str) -> None:
+ """Follow an edge to *target*. *edge_owner* is the object whose
+ ``next_block_label`` will be set to ``None`` when the target forms a
+ back-edge. *parent_label* is the block label that owns this edge
+ for logging."""
+ nonlocal found_cycle
+ if not target or target not in label_to_block:
+ return
+ if target in rec_stack:
+ is_branch = hasattr(edge_owner, "criteria")
+ LOG.warning(
+ "Copilot produced circular block chain, breaking cycle",
+ cycle_target=target,
+ broken_at=parent_label,
+ is_branch_condition=is_branch,
+ branch_expression=getattr(getattr(edge_owner, "criteria", None), "expression", None),
+ )
+ edge_owner.next_block_label = None
+ found_cycle = True
+ return
+ if target in visited:
+ return # merge point — not a cycle
+ _dfs(target)
+
+ def _dfs(label: str) -> None:
+ visited.add(label)
+ rec_stack.add(label)
+ block = label_to_block[label]
+
+ if isinstance(block, ConditionalBlockYAML):
+ for branch in block.branch_conditions:
+ _follow_edge(branch.next_block_label, branch, label)
+
+ _follow_edge(block.next_block_label, block, label)
+ rec_stack.discard(label)
+
+ if start_label in label_to_block:
+ _dfs(start_label)
+ return found_cycle
+
+
+def _find_terminal_label(
+ start_label: str,
+ label_to_block: dict[str, BlockYAML],
+ all_labels: set[str],
+) -> str | None:
+ """Find the terminal block by walking the main chain from start_label."""
+ visited: set[str] = set()
+ current: str | None = start_label
+ while current and current in label_to_block and current not in visited:
+ visited.add(current)
+ block = label_to_block[current]
+ if block.next_block_label is None or block.next_block_label not in all_labels:
+ return current
+ current = block.next_block_label
+ return None
+
+
+def _order_orphaned_blocks(
+ orphaned_labels: set[str],
+ label_to_block: dict[str, BlockYAML],
+ all_labels: set[str],
+ blocks: list[BlockYAML],
+) -> list[str]:
+ """Order orphaned blocks by following their internal next_block_label chains.
+
+ Multiple disconnected orphan sub-chains are concatenated in the order their
+ chain-start appears in the original blocks list.
+ """
+ pointed_to: set[str] = set()
+ for label in orphaned_labels:
+ block = label_to_block[label]
+ if block.next_block_label and block.next_block_label in orphaned_labels:
+ pointed_to.add(block.next_block_label)
+
+ # Chain starts are orphans not pointed to by another orphan.
+ # Preserve original array order for deterministic stitching.
+ chain_starts = [b.label for b in blocks if b.label in orphaned_labels and b.label not in pointed_to]
+
+ # If all orphans point to each other (cycle), pick the first in array order.
+ if not chain_starts:
+ chain_starts = [next(b.label for b in blocks if b.label in orphaned_labels)]
+
+ ordered: list[str] = []
+ visited: set[str] = set()
+ for start in chain_starts:
+ current: str | None = start
+ while current and current in orphaned_labels and current not in visited:
+ visited.add(current)
+ ordered.append(current)
+ current = label_to_block[current].next_block_label
+
+ # Append any remaining orphans not reached (multiple cycles).
+ for block in blocks:
+ if block.label in orphaned_labels and block.label not in visited:
+ ordered.append(block.label)
+
+ # Re-link the orphan chain so it forms a single connected path.
+ # This may overwrite an orphan's original next_block_label that pointed to a
+ # reachable block (a merge/join pattern). Log when this happens.
+ for i in range(len(ordered) - 1):
+ old_target = label_to_block[ordered[i]].next_block_label
+ new_target = ordered[i + 1]
+ if old_target and old_target != new_target and old_target not in orphaned_labels:
+ LOG.info(
+ "Orphan re-link overwrites cross-chain reference",
+ block=ordered[i],
+ old_target=old_target,
+ new_target=new_target,
+ )
+ label_to_block[ordered[i]].next_block_label = new_target
+ if ordered:
+ old_last_target = label_to_block[ordered[-1]].next_block_label
+ if old_last_target and old_last_target not in orphaned_labels:
+ LOG.info(
+ "Orphan chain terminal overwrites cross-chain reference",
+ block=ordered[-1],
+ old_target=old_last_target,
+ )
+ label_to_block[ordered[-1]].next_block_label = None
+
+ return ordered
+
+
+def _repair_next_block_label_chain(blocks: list[BlockYAML]) -> None:
+ """Ensure all top-level blocks form a single acyclic chain from blocks[0].
+
+ Repairs two classes of LLM mistakes:
+ 1. Circular references — breaks cycles so the chain has a proper terminal block.
+ 2. Disconnected paths — stitches orphaned blocks onto the end of the reachable chain.
+
+ Recursively repairs nested loop block ``loop_blocks`` at all depths.
+ Mutates *blocks* in place.
+ """
+ if len(blocks) <= 1:
+ # Still recurse into loop_blocks even for single-block lists
+ for block in blocks:
+ if isinstance(block, (ForLoopBlockYAML, WhileLoopBlockYAML)) and block.loop_blocks:
+ _repair_next_block_label_chain(block.loop_blocks)
+ return
+
+ # Warn on duplicate labels — the dict comprehension silently keeps the last
+ # occurrence, so earlier blocks with the same label become invisible.
+ seen_labels: set[str] = set()
+ for block in blocks:
+ if block.label in seen_labels:
+ LOG.warning("Copilot produced duplicate block label", label=block.label)
+ seen_labels.add(block.label)
+
+ label_to_block: dict[str, BlockYAML] = {block.label: block for block in blocks}
+ all_labels = set(label_to_block.keys())
+
+ # Phase 1: break any circular references reachable from the first block.
+ # Note: cycles among orphaned blocks (unreachable from blocks[0]) are handled
+ # implicitly by _order_orphaned_blocks via its visited set and re-linking logic.
+ _break_cycles(blocks[0].label, label_to_block)
+
+ # Phase 2: find orphaned (unreachable) blocks and stitch them to the end.
+ reachable: set[str] = set()
+ _collect_reachable(blocks[0].label, label_to_block, reachable)
+
+ orphaned_labels = all_labels - reachable
+ if orphaned_labels:
+ LOG.warning(
+ "Copilot produced disconnected workflow blocks, repairing chain",
+ orphaned_labels=sorted(orphaned_labels),
+ reachable_labels=sorted(reachable),
+ )
+
+ terminal_label = _find_terminal_label(blocks[0].label, label_to_block, all_labels)
+ ordered_orphan_labels = _order_orphaned_blocks(orphaned_labels, label_to_block, all_labels, blocks)
+
+ if terminal_label and ordered_orphan_labels:
+ label_to_block[terminal_label].next_block_label = ordered_orphan_labels[0]
+
+ # Phase 3: recursively repair nested loop block ``loop_blocks``.
+ for block in blocks:
+ if isinstance(block, (ForLoopBlockYAML, WhileLoopBlockYAML)) and block.loop_blocks:
+ _repair_next_block_label_chain(block.loop_blocks)
+
+
+def _normalize_copilot_yaml(workflow_yaml: str) -> WorkflowCreateYAMLRequest:
+ parsed_yaml = safe_load_no_dates(workflow_yaml)
+
+ # Fixing trivial common LLM mistakes; non-dict YAML falls through to model_validate.
+ if isinstance(parsed_yaml, dict):
+ # title is schema-required; coerce rather than force a self-healing round-trip.
+ parsed_yaml.setdefault("title", "")
+ _canonicalize_copilot_proxy_location(parsed_yaml)
+ workflow_definition = parsed_yaml.get("workflow_definition", None)
+ if workflow_definition:
+ _canonicalize_copilot_block_type_aliases(workflow_definition)
+ blocks = workflow_definition.get("blocks", []) or []
+ for block in blocks:
+ block["title"] = block.get("title", "")
+
+ workflow_yaml_request = WorkflowCreateYAMLRequest.model_validate(parsed_yaml)
+
+ # Post-processing
+ for block in workflow_yaml_request.workflow_definition.blocks:
+ if isinstance(block, LoginBlockYAML) and not block.navigation_goal:
+ block.navigation_goal = DEFAULT_LOGIN_PROMPT
+
+ workflow_yaml_request.workflow_definition.parameters = [
+ p for p in workflow_yaml_request.workflow_definition.parameters if p.parameter_type != ParameterType.OUTPUT
+ ]
+
+ _repair_next_block_label_chain(workflow_yaml_request.workflow_definition.blocks)
+
+ return workflow_yaml_request
+
+
+def _yaml_bool_setting(workflow_yaml: str | None, setting_name: str) -> bool | None:
+ if not workflow_yaml:
+ return None
+ try:
+ parsed = yaml.safe_load(workflow_yaml)
+ except yaml.YAMLError:
+ return None
+ if not isinstance(parsed, dict):
+ return None
+ value = parsed.get(setting_name)
+ return value if isinstance(value, bool) else None
+
+
+def _yaml_enable_self_healing(workflow_yaml: str | None) -> bool | None:
+ return _yaml_bool_setting(workflow_yaml, "enable_self_healing")
+
+
+def _yaml_pin_saved_session_ip(workflow_yaml: str | None) -> bool | None:
+ return _yaml_bool_setting(workflow_yaml, "pin_saved_session_ip")
+
+
+async def _process_workflow_yaml(
+ workflow_id: str,
+ workflow_permanent_id: str,
+ organization_id: str,
+ workflow_yaml: str,
+ settings_fallback_yaml: str | None = None,
+) -> Workflow:
+ # Single seam every copilot YAML->Workflow conversion passes through, so code
+ # blocks get their plain-view steps regardless of which path produced the YAML
+ # (the update_workflow tool derives them upstream; the inline REPLACE_WORKFLOW
+ # fallbacks would otherwise surface "No steps yet").
+ workflow_yaml = derive_code_block_steps_in_yaml(workflow_yaml)
+ workflow_yaml_request = _normalize_copilot_yaml(workflow_yaml)
+
+ updated_workflow_definition = convert_workflow_definition(
+ workflow_definition_yaml=workflow_yaml_request.workflow_definition,
+ workflow_id=workflow_id,
+ )
+
+ enable_self_healing = workflow_yaml_request.enable_self_healing
+ if enable_self_healing is None:
+ # Copilot YAML routinely omits settings it didn't touch; omission must inherit —
+ # the canonical-persist comparison would otherwise read the schema default as an
+ # explicit disable. The submitted draft YAML wins over persisted state so an
+ # unsaved editor toggle survives an unrelated copilot edit. A persisted-lookup
+ # failure propagates: failing the save is safer than writing an implicit disable.
+ enable_self_healing = _yaml_enable_self_healing(settings_fallback_yaml)
+
+ pin_saved_session_ip = _yaml_pin_saved_session_ip(workflow_yaml)
+ if pin_saved_session_ip is None:
+ pin_saved_session_ip = _yaml_pin_saved_session_ip(settings_fallback_yaml)
+
+ current_workflow: Workflow | None = None
+ if enable_self_healing is None or pin_saved_session_ip is None:
+ try:
+ current_workflow = await app.WORKFLOW_SERVICE.get_workflow_by_permanent_id(
+ workflow_permanent_id=workflow_permanent_id,
+ organization_id=organization_id,
+ )
+ except WorkflowNotFound:
+ current_workflow = None
+ if enable_self_healing is None:
+ enable_self_healing = bool(current_workflow and getattr(current_workflow, "enable_self_healing", False))
+ if pin_saved_session_ip is None:
+ pin_saved_session_ip = bool(current_workflow and getattr(current_workflow, "pin_saved_session_ip", False))
+
+ now = datetime.now(timezone.utc)
+ return Workflow(
+ workflow_id=workflow_id,
+ organization_id=organization_id,
+ title=workflow_yaml_request.title or "",
+ workflow_permanent_id=workflow_permanent_id,
+ version=1,
+ is_saved_task=workflow_yaml_request.is_saved_task,
+ description=workflow_yaml_request.description,
+ workflow_definition=updated_workflow_definition,
+ proxy_location=workflow_yaml_request.proxy_location,
+ webhook_callback_url=workflow_yaml_request.webhook_callback_url,
+ totp_verification_url=workflow_yaml_request.totp_verification_url,
+ totp_identifier=workflow_yaml_request.totp_identifier,
+ persist_browser_session=workflow_yaml_request.persist_browser_session or False,
+ pin_saved_session_ip=pin_saved_session_ip,
+ browser_profile_id=workflow_yaml_request.browser_profile_id,
+ browser_profile_key=workflow_yaml_request.browser_profile_key,
+ model=workflow_yaml_request.model,
+ max_screenshot_scrolls=workflow_yaml_request.max_screenshot_scrolls,
+ extra_http_headers=workflow_yaml_request.extra_http_headers,
+ cdp_connect_headers=workflow_yaml_request.cdp_connect_headers,
+ run_with=workflow_yaml_request.run_with,
+ ai_fallback=workflow_yaml_request.ai_fallback,
+ cache_key=workflow_yaml_request.cache_key,
+ adaptive_caching=workflow_yaml_request.adaptive_caching,
+ enable_self_healing=enable_self_healing,
+ code_version=workflow_yaml_request.code_version,
+ run_sequentially=workflow_yaml_request.run_sequentially,
+ sequential_key=workflow_yaml_request.sequential_key,
+ created_at=now,
+ modified_at=now,
+ )
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 432f82139..58cf33c62 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -18,8 +18,6 @@ from sse_starlette import EventSourceResponse
from skyvern import analytics
from skyvern.config import settings
-from skyvern.constants import DEFAULT_LOGIN_PROMPT
-from skyvern.exceptions import WorkflowNotFound
from skyvern.forge import app
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
@@ -27,8 +25,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityType
from skyvern.forge.sdk.copilot.agent import run_copilot_agent
from skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write, resolve_copilot_created_by_stamp
-from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
-from skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_steps, derive_code_block_steps_in_yaml
+from skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_steps
from skyvern.forge.sdk.copilot.completion_criteria_store import (
CRITERIA_SET_STATUS_ACTIVE,
StoredCriteriaSet,
@@ -60,6 +57,9 @@ from skyvern.forge.sdk.copilot.turn_outcome import (
build_minimal_turn_outcome,
with_copilot_code_mode_metadata,
)
+from skyvern.forge.sdk.copilot.workflow_yaml import _normalize_copilot_yaml as _normalize_copilot_yaml
+from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml as _process_workflow_yaml
+from skyvern.forge.sdk.copilot.workflow_yaml import _repair_next_block_label_chain as _repair_next_block_label_chain
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream, FastAPIEventSourceStream
from skyvern.forge.sdk.routes.routers import base_router
@@ -89,14 +89,7 @@ from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
from skyvern.forge.sdk.workflow.models.parameter import ParameterType
from skyvern.forge.sdk.workflow.models.workflow import Workflow
from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
-from skyvern.schemas.runs import ProxyLocation
from skyvern.schemas.workflows import (
- BlockYAML,
- BranchConditionYAML,
- ConditionalBlockYAML,
- ForLoopBlockYAML,
- LoginBlockYAML,
- WhileLoopBlockYAML,
WorkflowCreateYAMLRequest,
WorkflowDefinitionYAML,
)
@@ -118,63 +111,6 @@ ALLOWED_WORKFLOW_COPILOT_AUDIO_CONTENT_TYPES = {
LOG = structlog.get_logger()
-def _proxy_location_alias_key(value: str) -> str:
- return "_".join(value.strip().upper().replace("-", "_").split())
-
-
-def _build_copilot_proxy_location_aliases() -> dict[str, str]:
- aliases: dict[str, str] = {}
-
- def add(alias: str, proxy_location: ProxyLocation) -> None:
- aliases[_proxy_location_alias_key(alias)] = proxy_location.value
-
- for proxy_location in ProxyLocation:
- add(proxy_location.name, proxy_location)
- add(proxy_location.value, proxy_location)
-
- for proxy_location in ProxyLocation.residential_country_locations():
- add(ProxyLocation.get_country_code(proxy_location), proxy_location)
-
- add("USA", ProxyLocation.RESIDENTIAL)
- add("United States", ProxyLocation.RESIDENTIAL)
- add("United States of America", ProxyLocation.RESIDENTIAL)
- add("RESIDENTIAL_US", ProxyLocation.RESIDENTIAL)
- add("UK", ProxyLocation.RESIDENTIAL_GB)
- add("United Kingdom", ProxyLocation.RESIDENTIAL_GB)
-
- return aliases
-
-
-_COPILOT_PROXY_LOCATION_ALIASES = _build_copilot_proxy_location_aliases()
-
-
-def _canonicalize_copilot_proxy_location(parsed_yaml: dict[str, Any]) -> None:
- if "proxy_location" not in parsed_yaml:
- return
-
- proxy_location = parsed_yaml.get("proxy_location")
- if not isinstance(proxy_location, str):
- return
-
- canonical = _COPILOT_PROXY_LOCATION_ALIASES.get(_proxy_location_alias_key(proxy_location))
- if canonical is None:
- return
-
- parsed_yaml["proxy_location"] = canonical
-
-
-def _canonicalize_copilot_block_type_aliases(value: Any) -> None:
- if isinstance(value, dict):
- block_type = value.get("block_type")
- if isinstance(block_type, str):
- value["block_type"] = normalize_copilot_block_type_alias(block_type)
- for child in value.values():
- _canonicalize_copilot_block_type_aliases(child)
- elif isinstance(value, list):
- for item in value:
- _canonicalize_copilot_block_type_aliases(item)
-
-
async def _resolve_copilot_agent_handler(
workflow_permanent_id: str,
organization_id: str,
@@ -1314,366 +1250,6 @@ async def _auto_correct_workflow_yaml(
return action_data.get("workflow_yaml", workflow_yaml)
-def _collect_reachable(
- start_label: str,
- label_to_block: dict[str, BlockYAML],
- reachable: set[str],
-) -> None:
- """Walk the next_block_label chain from start_label, collecting all reachable labels.
-
- For conditional blocks, also follows branch target chains recursively.
-
- The ``current not in reachable`` loop guard means the main-chain walk
- stops early if we hit a node already collected via a branch recursion.
- This is correct — those downstream nodes and their successors are
- already in ``reachable`` — but callers should be aware of the coupling.
- """
- current: str | None = start_label
- while current and current in label_to_block and current not in reachable:
- reachable.add(current)
- block = label_to_block[current]
- if isinstance(block, ConditionalBlockYAML):
- for branch in block.branch_conditions:
- if branch.next_block_label and branch.next_block_label not in reachable:
- _collect_reachable(branch.next_block_label, label_to_block, reachable)
- current = block.next_block_label
-
-
-def _break_cycles(
- start_label: str,
- label_to_block: dict[str, BlockYAML],
-) -> bool:
- """Detect and break circular references in the block chain using DFS.
-
- Uses a recursion stack to distinguish true back-edges (cycles) from merge
- points (two branches converging on the same block). When a back-edge is
- found the offending ``next_block_label`` is set to ``None``, breaking the
- cycle. Handles both the main chain and conditional branch chains.
-
- Note: this function operates on a single level of blocks. It does **not**
- recurse into ``ForLoopBlockYAML.loop_blocks``; nested loops are handled
- by the recursive ``_repair_next_block_label_chain`` call in Phase 3.
-
- Returns True if at least one cycle was broken.
- """
- visited: set[str] = set()
- rec_stack: set[str] = set()
- found_cycle = False
-
- def _follow_edge(target: str | None, edge_owner: BlockYAML | BranchConditionYAML, parent_label: str) -> None:
- """Follow an edge to *target*. *edge_owner* is the object whose
- ``next_block_label`` will be set to ``None`` when the target forms a
- back-edge. *parent_label* is the block label that owns this edge
- for logging."""
- nonlocal found_cycle
- if not target or target not in label_to_block:
- return
- if target in rec_stack:
- is_branch = hasattr(edge_owner, "criteria")
- LOG.warning(
- "Copilot produced circular block chain, breaking cycle",
- cycle_target=target,
- broken_at=parent_label,
- is_branch_condition=is_branch,
- branch_expression=getattr(getattr(edge_owner, "criteria", None), "expression", None),
- )
- edge_owner.next_block_label = None
- found_cycle = True
- return
- if target in visited:
- return # merge point — not a cycle
- _dfs(target)
-
- def _dfs(label: str) -> None:
- visited.add(label)
- rec_stack.add(label)
- block = label_to_block[label]
-
- if isinstance(block, ConditionalBlockYAML):
- for branch in block.branch_conditions:
- _follow_edge(branch.next_block_label, branch, label)
-
- _follow_edge(block.next_block_label, block, label)
- rec_stack.discard(label)
-
- if start_label in label_to_block:
- _dfs(start_label)
- return found_cycle
-
-
-def _find_terminal_label(
- start_label: str,
- label_to_block: dict[str, BlockYAML],
- all_labels: set[str],
-) -> str | None:
- """Find the terminal block by walking the main chain from start_label."""
- visited: set[str] = set()
- current: str | None = start_label
- while current and current in label_to_block and current not in visited:
- visited.add(current)
- block = label_to_block[current]
- if block.next_block_label is None or block.next_block_label not in all_labels:
- return current
- current = block.next_block_label
- return None
-
-
-def _order_orphaned_blocks(
- orphaned_labels: set[str],
- label_to_block: dict[str, BlockYAML],
- all_labels: set[str],
- blocks: list[BlockYAML],
-) -> list[str]:
- """Order orphaned blocks by following their internal next_block_label chains.
-
- Multiple disconnected orphan sub-chains are concatenated in the order their
- chain-start appears in the original blocks list.
- """
- pointed_to: set[str] = set()
- for label in orphaned_labels:
- block = label_to_block[label]
- if block.next_block_label and block.next_block_label in orphaned_labels:
- pointed_to.add(block.next_block_label)
-
- # Chain starts are orphans not pointed to by another orphan.
- # Preserve original array order for deterministic stitching.
- chain_starts = [b.label for b in blocks if b.label in orphaned_labels and b.label not in pointed_to]
-
- # If all orphans point to each other (cycle), pick the first in array order.
- if not chain_starts:
- chain_starts = [next(b.label for b in blocks if b.label in orphaned_labels)]
-
- ordered: list[str] = []
- visited: set[str] = set()
- for start in chain_starts:
- current: str | None = start
- while current and current in orphaned_labels and current not in visited:
- visited.add(current)
- ordered.append(current)
- current = label_to_block[current].next_block_label
-
- # Append any remaining orphans not reached (multiple cycles).
- for block in blocks:
- if block.label in orphaned_labels and block.label not in visited:
- ordered.append(block.label)
-
- # Re-link the orphan chain so it forms a single connected path.
- # This may overwrite an orphan's original next_block_label that pointed to a
- # reachable block (a merge/join pattern). Log when this happens.
- for i in range(len(ordered) - 1):
- old_target = label_to_block[ordered[i]].next_block_label
- new_target = ordered[i + 1]
- if old_target and old_target != new_target and old_target not in orphaned_labels:
- LOG.info(
- "Orphan re-link overwrites cross-chain reference",
- block=ordered[i],
- old_target=old_target,
- new_target=new_target,
- )
- label_to_block[ordered[i]].next_block_label = new_target
- if ordered:
- old_last_target = label_to_block[ordered[-1]].next_block_label
- if old_last_target and old_last_target not in orphaned_labels:
- LOG.info(
- "Orphan chain terminal overwrites cross-chain reference",
- block=ordered[-1],
- old_target=old_last_target,
- )
- label_to_block[ordered[-1]].next_block_label = None
-
- return ordered
-
-
-def _repair_next_block_label_chain(blocks: list[BlockYAML]) -> None:
- """Ensure all top-level blocks form a single acyclic chain from blocks[0].
-
- Repairs two classes of LLM mistakes:
- 1. Circular references — breaks cycles so the chain has a proper terminal block.
- 2. Disconnected paths — stitches orphaned blocks onto the end of the reachable chain.
-
- Recursively repairs nested loop block ``loop_blocks`` at all depths.
- Mutates *blocks* in place.
- """
- if len(blocks) <= 1:
- # Still recurse into loop_blocks even for single-block lists
- for block in blocks:
- if isinstance(block, (ForLoopBlockYAML, WhileLoopBlockYAML)) and block.loop_blocks:
- _repair_next_block_label_chain(block.loop_blocks)
- return
-
- # Warn on duplicate labels — the dict comprehension silently keeps the last
- # occurrence, so earlier blocks with the same label become invisible.
- seen_labels: set[str] = set()
- for block in blocks:
- if block.label in seen_labels:
- LOG.warning("Copilot produced duplicate block label", label=block.label)
- seen_labels.add(block.label)
-
- label_to_block: dict[str, BlockYAML] = {block.label: block for block in blocks}
- all_labels = set(label_to_block.keys())
-
- # Phase 1: break any circular references reachable from the first block.
- # Note: cycles among orphaned blocks (unreachable from blocks[0]) are handled
- # implicitly by _order_orphaned_blocks via its visited set and re-linking logic.
- _break_cycles(blocks[0].label, label_to_block)
-
- # Phase 2: find orphaned (unreachable) blocks and stitch them to the end.
- reachable: set[str] = set()
- _collect_reachable(blocks[0].label, label_to_block, reachable)
-
- orphaned_labels = all_labels - reachable
- if orphaned_labels:
- LOG.warning(
- "Copilot produced disconnected workflow blocks, repairing chain",
- orphaned_labels=sorted(orphaned_labels),
- reachable_labels=sorted(reachable),
- )
-
- terminal_label = _find_terminal_label(blocks[0].label, label_to_block, all_labels)
- ordered_orphan_labels = _order_orphaned_blocks(orphaned_labels, label_to_block, all_labels, blocks)
-
- if terminal_label and ordered_orphan_labels:
- label_to_block[terminal_label].next_block_label = ordered_orphan_labels[0]
-
- # Phase 3: recursively repair nested loop block ``loop_blocks``.
- for block in blocks:
- if isinstance(block, (ForLoopBlockYAML, WhileLoopBlockYAML)) and block.loop_blocks:
- _repair_next_block_label_chain(block.loop_blocks)
-
-
-def _normalize_copilot_yaml(workflow_yaml: str) -> WorkflowCreateYAMLRequest:
- parsed_yaml = safe_load_no_dates(workflow_yaml)
-
- # Fixing trivial common LLM mistakes; non-dict YAML falls through to model_validate.
- if isinstance(parsed_yaml, dict):
- # title is schema-required; coerce rather than force a self-healing round-trip.
- parsed_yaml.setdefault("title", "")
- _canonicalize_copilot_proxy_location(parsed_yaml)
- workflow_definition = parsed_yaml.get("workflow_definition", None)
- if workflow_definition:
- _canonicalize_copilot_block_type_aliases(workflow_definition)
- blocks = workflow_definition.get("blocks", []) or []
- for block in blocks:
- block["title"] = block.get("title", "")
-
- workflow_yaml_request = WorkflowCreateYAMLRequest.model_validate(parsed_yaml)
-
- # Post-processing
- for block in workflow_yaml_request.workflow_definition.blocks:
- if isinstance(block, LoginBlockYAML) and not block.navigation_goal:
- block.navigation_goal = DEFAULT_LOGIN_PROMPT
-
- workflow_yaml_request.workflow_definition.parameters = [
- p for p in workflow_yaml_request.workflow_definition.parameters if p.parameter_type != ParameterType.OUTPUT
- ]
-
- _repair_next_block_label_chain(workflow_yaml_request.workflow_definition.blocks)
-
- return workflow_yaml_request
-
-
-def _yaml_bool_setting(workflow_yaml: str | None, setting_name: str) -> bool | None:
- if not workflow_yaml:
- return None
- try:
- parsed = yaml.safe_load(workflow_yaml)
- except yaml.YAMLError:
- return None
- if not isinstance(parsed, dict):
- return None
- value = parsed.get(setting_name)
- return value if isinstance(value, bool) else None
-
-
-def _yaml_enable_self_healing(workflow_yaml: str | None) -> bool | None:
- return _yaml_bool_setting(workflow_yaml, "enable_self_healing")
-
-
-def _yaml_pin_saved_session_ip(workflow_yaml: str | None) -> bool | None:
- return _yaml_bool_setting(workflow_yaml, "pin_saved_session_ip")
-
-
-async def _process_workflow_yaml(
- workflow_id: str,
- workflow_permanent_id: str,
- organization_id: str,
- workflow_yaml: str,
- settings_fallback_yaml: str | None = None,
-) -> Workflow:
- # Single seam every copilot YAML->Workflow conversion passes through, so code
- # blocks get their plain-view steps regardless of which path produced the YAML
- # (the update_workflow tool derives them upstream; the inline REPLACE_WORKFLOW
- # fallbacks would otherwise surface "No steps yet").
- workflow_yaml = derive_code_block_steps_in_yaml(workflow_yaml)
- workflow_yaml_request = _normalize_copilot_yaml(workflow_yaml)
-
- updated_workflow_definition = convert_workflow_definition(
- workflow_definition_yaml=workflow_yaml_request.workflow_definition,
- workflow_id=workflow_id,
- )
-
- enable_self_healing = workflow_yaml_request.enable_self_healing
- if enable_self_healing is None:
- # Copilot YAML routinely omits settings it didn't touch; omission must inherit —
- # the canonical-persist comparison would otherwise read the schema default as an
- # explicit disable. The submitted draft YAML wins over persisted state so an
- # unsaved editor toggle survives an unrelated copilot edit. A persisted-lookup
- # failure propagates: failing the save is safer than writing an implicit disable.
- enable_self_healing = _yaml_enable_self_healing(settings_fallback_yaml)
-
- pin_saved_session_ip = _yaml_pin_saved_session_ip(workflow_yaml)
- if pin_saved_session_ip is None:
- pin_saved_session_ip = _yaml_pin_saved_session_ip(settings_fallback_yaml)
-
- current_workflow: Workflow | None = None
- if enable_self_healing is None or pin_saved_session_ip is None:
- try:
- current_workflow = await app.WORKFLOW_SERVICE.get_workflow_by_permanent_id(
- workflow_permanent_id=workflow_permanent_id,
- organization_id=organization_id,
- )
- except WorkflowNotFound:
- current_workflow = None
- if enable_self_healing is None:
- enable_self_healing = bool(current_workflow and getattr(current_workflow, "enable_self_healing", False))
- if pin_saved_session_ip is None:
- pin_saved_session_ip = bool(current_workflow and getattr(current_workflow, "pin_saved_session_ip", False))
-
- now = datetime.now(timezone.utc)
- return Workflow(
- workflow_id=workflow_id,
- organization_id=organization_id,
- title=workflow_yaml_request.title or "",
- workflow_permanent_id=workflow_permanent_id,
- version=1,
- is_saved_task=workflow_yaml_request.is_saved_task,
- description=workflow_yaml_request.description,
- workflow_definition=updated_workflow_definition,
- proxy_location=workflow_yaml_request.proxy_location,
- webhook_callback_url=workflow_yaml_request.webhook_callback_url,
- totp_verification_url=workflow_yaml_request.totp_verification_url,
- totp_identifier=workflow_yaml_request.totp_identifier,
- persist_browser_session=workflow_yaml_request.persist_browser_session or False,
- pin_saved_session_ip=pin_saved_session_ip,
- browser_profile_id=workflow_yaml_request.browser_profile_id,
- browser_profile_key=workflow_yaml_request.browser_profile_key,
- model=workflow_yaml_request.model,
- max_screenshot_scrolls=workflow_yaml_request.max_screenshot_scrolls,
- extra_http_headers=workflow_yaml_request.extra_http_headers,
- cdp_connect_headers=workflow_yaml_request.cdp_connect_headers,
- run_with=workflow_yaml_request.run_with,
- ai_fallback=workflow_yaml_request.ai_fallback,
- cache_key=workflow_yaml_request.cache_key,
- adaptive_caching=workflow_yaml_request.adaptive_caching,
- enable_self_healing=enable_self_healing,
- code_version=workflow_yaml_request.code_version,
- run_sequentially=workflow_yaml_request.run_sequentially,
- sequential_key=workflow_yaml_request.sequential_key,
- created_at=now,
- modified_at=now,
- )
-
-
def _blockless_submission_fallback(
*,
proposed_workflow: dict[str, Any] | None,
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index e851ad6fe..9323edce8 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -1688,20 +1688,42 @@ def test_validation_classification_grader_credits_repeated_matching_boolean_valu
),
],
)
-def test_validation_classification_grader_fails_closed_for_incomplete_contract(
+def test_validation_classification_grader_abstains_on_incomplete_contract(
criterion: CompletionCriterion,
) -> None:
+ # SKY-12340: an incomplete typed contract (missing output key or expected value) cannot grade
+ # anything, so it degrades to a non-sinking abstention like the definition grader instead of a
+ # sinking unsatisfied/no_evidence verdict that would disown a delivered sibling output.
snapshot = RunEvidenceSnapshot(block_outputs={"classify_path": {"path_classification": "login_gated"}})
verdicts = grade_validation_classification_criteria([criterion], snapshot)
assert len(verdicts) == 1
assert verdicts[0].criterion_id == "c_validation"
- assert verdicts[0].state == "unsatisfied"
- assert verdicts[0].reason_code == "no_evidence"
+ assert verdicts[0].state == "unknown"
+ assert verdicts[0].reason_code == "validation_classification_incomplete_contract"
assert verdicts[0].missing_evidence == "incomplete typed classification contract"
+def test_incomplete_validation_classification_abstention_does_not_sink_confirmed_run() -> None:
+ # Direction 1 (real delivery credits): an incomplete-contract abstention must not veto a run
+ # whose delivered sibling output is confirmed.
+ incomplete = CriterionVerdict(
+ criterion_id="c0", state="unknown", reason_code="validation_classification_incomplete_contract"
+ )
+ confirmed = CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms")
+ assert _mixed(incomplete, confirmed).is_fully_satisfied() is True
+
+
+def test_incomplete_validation_classification_abstention_alone_never_satisfies() -> None:
+ # Direction 2 (fail-closed): a value-less classification criterion can never manufacture success
+ # on its own — an abstention-only result is not fully satisfied.
+ incomplete = CriterionVerdict(
+ criterion_id="c0", state="unknown", reason_code="validation_classification_incomplete_contract"
+ )
+ assert _mixed(incomplete).is_fully_satisfied() is False
+
+
@pytest.mark.parametrize(
("payload", "reason_code"),
[
@@ -5054,10 +5076,13 @@ async def test_validation_classification_missing_or_prose_only_evidence_cannot_b
],
)
@pytest.mark.asyncio
-async def test_validation_classification_incomplete_contract_cannot_be_judge_approved(
+async def test_validation_classification_incomplete_contract_abstains_without_blocking_satisfied_sibling(
monkeypatch: pytest.MonkeyPatch,
criterion: CompletionCriterion,
) -> None:
+ # SKY-12340: the incomplete contract still cannot be judge-approved (its deterministic verdict
+ # stays a non-sinking abstention, never satisfied), but it no longer sinks a run whose genuine
+ # sibling output is satisfied.
calls = 0
async def handler(**_: object) -> dict:
@@ -5089,12 +5114,14 @@ async def test_validation_classification_incomplete_contract_cannot_be_judge_app
assert calls == 1
assert verification is not None
- assert verification.is_fully_satisfied() is False
verdict_by_id = {verdict.criterion_id: verdict for verdict in verification.verdicts}
- assert verdict_by_id["c_validation"].state == "unsatisfied"
- assert verdict_by_id["c_validation"].reason_code == "no_evidence"
+ # Judge said satisfied, but the incomplete contract abstains deterministically — never credited.
+ assert verdict_by_id["c_validation"].state == "unknown"
+ assert verdict_by_id["c_validation"].reason_code == "validation_classification_incomplete_contract"
assert verdict_by_id["c_validation"].missing_evidence == "incomplete typed classification contract"
assert verdict_by_id["c_other"].satisfied is True
+ # The abstention does not disown the genuinely satisfied sibling.
+ assert verification.is_fully_satisfied() is True
@pytest.mark.asyncio
@@ -5916,10 +5943,12 @@ async def test_page_observation_validation_classification_cannot_be_judge_approv
],
)
@pytest.mark.asyncio
-async def test_page_observation_validation_classification_incomplete_contract_cannot_be_judge_approved(
+async def test_page_observation_validation_classification_incomplete_contract_abstains_without_blocking_sibling(
monkeypatch: pytest.MonkeyPatch,
criterion: CompletionCriterion,
) -> None:
+ # SKY-12340: same abstention on the page-observation path — the incomplete contract is never
+ # judge-credited, but no longer sinks a run whose genuine sibling is satisfied.
handler_calls = 0
async def handler(**_: object) -> dict:
@@ -5961,12 +5990,12 @@ async def test_page_observation_validation_classification_incomplete_contract_ca
assert handler_calls == 1
assert result is not None
- assert result.is_fully_satisfied() is False
verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}
- assert verdict_by_id["c_validation"].state == "unsatisfied"
- assert verdict_by_id["c_validation"].reason_code == "no_evidence"
+ assert verdict_by_id["c_validation"].state == "unknown"
+ assert verdict_by_id["c_validation"].reason_code == "validation_classification_incomplete_contract"
assert verdict_by_id["c_validation"].missing_evidence == "incomplete typed classification contract"
assert verdict_by_id["c_page"].satisfied is True
+ assert result.is_fully_satisfied() is True
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_layering.py b/tests/unit/test_copilot_layering.py
new file mode 100644
index 000000000..ac2688944
--- /dev/null
+++ b/tests/unit/test_copilot_layering.py
@@ -0,0 +1,16 @@
+"""Layering guards: the copilot domain package must not depend on the HTTP routes layer."""
+
+import subprocess
+import sys
+
+
+def test_importing_copilot_tools_does_not_load_routes() -> None:
+ script = (
+ "import sys\n"
+ "import skyvern.forge.sdk.copilot.tools\n"
+ "loaded = sorted(m for m in sys.modules if m.startswith('skyvern.forge.sdk.routes'))\n"
+ "assert not loaded, f'copilot.tools pulled in routes modules: {loaded}'\n"
+ )
+ # Timeout so a pathological import hang fails loudly instead of stalling the run.
+ result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=120)
+ assert result.returncode == 0, f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
diff --git a/tests/unit/test_workflow_copilot_chain_repair.py b/tests/unit/test_workflow_copilot_chain_repair.py
index a849788a2..820cfb5c3 100644
--- a/tests/unit/test_workflow_copilot_chain_repair.py
+++ b/tests/unit/test_workflow_copilot_chain_repair.py
@@ -4,7 +4,7 @@ Ensures that _repair_next_block_label_chain correctly stitches orphaned blocks
back into the reachable workflow chain when the LLM produces disconnected paths.
"""
-from skyvern.forge.sdk.routes.workflow_copilot import (
+from skyvern.forge.sdk.copilot.workflow_yaml import (
_break_cycles,
_collect_reachable,
_find_terminal_label,