Skip to content

Automate browser based workflows with AI

feat(submission): shadow-mode deterministic submission signal behind default-off flag (SKY-12162) (#7281)

Marc Kelechava31e6bdacda3e
diff --git a/skyvern/config.py b/skyvern/config.py
index 171ef52da..a2d4900e8 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -145,6 +145,9 @@ class Settings(BaseSettings):
# Static kill-switch for fail-fast shadow observability. Per-org rollout is the
# PostHog flag FAIL_FAST_SHADOW; this only force-enables it everywhere (local/testing).
FAIL_FAST_SHADOW: bool = False
+ # Default-off telemetry for deterministic submission signals; enabling it only schedules
+ # fire-and-forget shadow evaluation and does not change run behavior.
+ SKYVERN_SUBMISSION_SIGNAL_SHADOW: bool = False
# Global kill-switch for select/autocomplete shadow-match observability (LLM-vs-deterministic
# agreement logging). Not per-org; set false to silence the logs everywhere.
SKYVERN_SELECT_SHADOW_MATCH: bool = True
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index ef02e1fe9..358cafa68 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -108,6 +108,7 @@ from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskResponse, TaskStatus
from skyvern.forge.sdk.schemas.totp_codes import OTPType
+from skyvern.forge.sdk.submission import shadow as submission_shadow
from skyvern.forge.sdk.trace import VerificationTrigger, apply_context_attrs, traced
from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import (
@@ -5006,6 +5007,14 @@ class ForgeAgent:
_use_bundling = _ctx.use_artifact_bundling if _ctx else False
har_data = await app.BROWSER_MANAGER.get_har_data(task_id=task.task_id, browser_state=browser_state)
+ if settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW:
+ submission_shadow.schedule_submission_signal_shadow(
+ har_data=har_data,
+ browser_state=browser_state,
+ last_step=last_step,
+ task=task,
+ browser_session_id=browser_session_id,
+ )
LOG.debug("Uploading har data", har_size=len(har_data))
browser_log = await app.BROWSER_MANAGER.get_browser_console_log(
diff --git a/skyvern/forge/sdk/submission/__init__.py b/skyvern/forge/sdk/submission/__init__.py
new file mode 100644
index 000000000..a8c90cb29
--- /dev/null
+++ b/skyvern/forge/sdk/submission/__init__.py
@@ -0,0 +1,47 @@
+from skyvern.forge.sdk.submission.models import (
+ BrowserPath,
+ CandidateWindow,
+ DownloadEvidence,
+ NetworkEvidence,
+ PageConfirmationEvidence,
+ SubmissionSignal,
+ SubmissionVerdict,
+ SubmitCandidate,
+ SubmitCandidateDetection,
+ TierAEvaluation,
+ TierBEvaluation,
+ TierBEvidence,
+ UrlTransitionEvidence,
+)
+from skyvern.forge.sdk.submission.verifier import (
+ build_candidate_windows,
+ classify_browser_path,
+ combine,
+ detect_submit_candidates,
+ evaluate_tier_a,
+ evaluate_tier_b,
+ find_latest_candidate_step_pair,
+)
+
+__all__ = [
+ "BrowserPath",
+ "CandidateWindow",
+ "DownloadEvidence",
+ "NetworkEvidence",
+ "PageConfirmationEvidence",
+ "SubmissionSignal",
+ "SubmissionVerdict",
+ "SubmitCandidate",
+ "SubmitCandidateDetection",
+ "TierAEvaluation",
+ "TierBEvaluation",
+ "TierBEvidence",
+ "UrlTransitionEvidence",
+ "build_candidate_windows",
+ "classify_browser_path",
+ "combine",
+ "detect_submit_candidates",
+ "evaluate_tier_a",
+ "evaluate_tier_b",
+ "find_latest_candidate_step_pair",
+]
diff --git a/skyvern/forge/sdk/submission/models.py b/skyvern/forge/sdk/submission/models.py
new file mode 100644
index 000000000..74117f103
--- /dev/null
+++ b/skyvern/forge/sdk/submission/models.py
@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from typing import TypeAlias
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class SubmissionSignal(StrEnum):
+ SUBMITTED_VERIFIED = "submitted_verified"
+ SUBMITTED_LIKELY = "submitted_likely"
+ NOT_SUBMITTED = "not_submitted"
+ NOT_EVALUATED = "not_evaluated"
+
+
+class BrowserPath(StrEnum):
+ SKYVERN_CREATED = "skyvern_created"
+ CDP_CONNECT = "cdp_connect"
+ VENDOR_REUSED = "vendor_reused"
+ SESSION_ATTACHED = "session_attached"
+ UNKNOWN = "unknown"
+
+
+class SubmitCandidate(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ action_id: str | None = None
+ task_id: str | None = None
+ step_id: str
+
+
+class SubmitCandidateDetection(BaseModel):
+ candidates: list[SubmitCandidate]
+ coordinate_click: bool = False
+
+ @property
+ def submit_intent_detected(self) -> bool:
+ return bool(self.candidates)
+
+
+class CandidateWindow(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ task_id: str | None = None
+ step_id: str
+ candidate_action_id: str | None = None
+ started_at: datetime
+ ended_at: datetime
+
+
+class NetworkEvidence(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ origin: str
+ method: str
+ status: int
+ started_at: datetime
+ correlated_step_id: str
+
+
+class PageConfirmationEvidence(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ phrase: str
+ value_sha256: str
+ absent_pre_submit: bool
+
+
+class UrlTransitionEvidence(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ from_origin: str
+ to_origin: str
+ path_changed: bool
+
+
+class DownloadEvidence(BaseModel):
+ model_config = ConfigDict(frozen=True)
+
+ file_count: int
+
+
+TierBEvidence: TypeAlias = PageConfirmationEvidence | UrlTransitionEvidence | DownloadEvidence
+
+
+class TierAEvaluation(BaseModel):
+ evidence: list[NetworkEvidence]
+ har_present: bool
+ har_parsed: bool
+ har_entry_count: int
+ ambiguous_entry_count: int = 0
+
+
+class TierBEvaluation(BaseModel):
+ evidence: list[TierBEvidence]
+ page_confirmation_evaluated: bool
+
+
+class SubmissionVerdict(BaseModel):
+ signal: SubmissionSignal
+ tier_a: list[NetworkEvidence]
+ tier_b: list[TierBEvidence]
+ submit_intent_detected: bool
+ har_present: bool
+ har_parsed: bool
+ har_entry_count: int
+ browser_path: BrowserPath
+ winning_step_id: str | None = None
+ capped: bool = False
+ notes: list[str] = Field(default_factory=list)
diff --git a/skyvern/forge/sdk/submission/shadow.py b/skyvern/forge/sdk/submission/shadow.py
new file mode 100644
index 000000000..c6eec1563
--- /dev/null
+++ b/skyvern/forge/sdk/submission/shadow.py
@@ -0,0 +1,390 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass
+
+import structlog
+
+from skyvern.config import settings
+from skyvern.forge import app
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.models import Step
+from skyvern.forge.sdk.schemas.tasks import Task
+from skyvern.forge.sdk.submission.models import BrowserPath, CandidateWindow, SubmissionVerdict, TierAEvaluation
+from skyvern.forge.sdk.submission.verifier import (
+ CandidateEvaluation,
+ build_candidate_windows,
+ classify_browser_path,
+ combine,
+ detect_submit_candidates,
+ evaluate_tier_a,
+ evaluate_tier_b,
+ find_candidate_step_pairs,
+)
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun
+from skyvern.services import service_utils
+from skyvern.webeye.actions.actions import Action
+from skyvern.webeye.actions.responses import ActionResult
+from skyvern.webeye.browser_state import BrowserState
+
+LOG = structlog.get_logger()
+
+_SHADOW_EVENT = "submission_signal_shadow"
+_ZIP_HEADER = b"PK\x03\x04"
+_PAGE_ARTIFACT_TYPES = (
+ ArtifactType.VISIBLE_ELEMENTS_TREE_IN_PROMPT,
+ ArtifactType.HTML_SCRAPE,
+)
+
+_PENDING_SUBMISSION_SHADOW_TASKS: set[asyncio.Task[None]] = set()
+_MAX_PENDING = 20
+_MAX_HAR_BYTES = 25_000_000
+
+
+@dataclass(frozen=True)
+class _LoadedContext:
+ tasks: list[Task]
+ steps: list[Step]
+ actions: list[Action]
+ organization_id: str
+ workflow_run_id: str | None
+ run_status: str
+ output_is_none: bool
+
+
+def _elapsed_ms(started_at: float) -> int:
+ return int((time.monotonic() - started_at) * 1000)
+
+
+def _prune_pending() -> None:
+ _PENDING_SUBMISSION_SHADOW_TASKS.difference_update(
+ {task for task in _PENDING_SUBMISSION_SHADOW_TASKS if task.done()}
+ )
+
+
+def _track(task: asyncio.Task[None]) -> asyncio.Task[None]:
+ _PENDING_SUBMISSION_SHADOW_TASKS.add(task)
+ task.add_done_callback(_PENDING_SUBMISSION_SHADOW_TASKS.discard)
+ return task
+
+
+def _merge_last_step(steps: list[Step], last_step: Step) -> list[Step]:
+ merged = [last_step if step.step_id == last_step.step_id else step for step in steps]
+ if all(step.step_id != last_step.step_id for step in steps):
+ merged.append(last_step)
+ return sorted(merged, key=lambda step: (step.created_at, step.order, step.retry_index))
+
+
+def _flatten_action_results(steps: list[Step]) -> list[ActionResult]:
+ results: list[ActionResult] = []
+ for step in steps:
+ if step.output is None:
+ continue
+ if step.output.actions_and_results:
+ results.extend(result for _, action_results in step.output.actions_and_results for result in action_results)
+ elif step.output.action_results:
+ results.extend(step.output.action_results)
+ return results
+
+
+async def _load_context(
+ *,
+ task: Task | None,
+ workflow_run: WorkflowRun | None,
+ last_step: Step,
+) -> _LoadedContext:
+ if task is None and workflow_run is None:
+ raise ValueError("task or workflow_run is required")
+ if task is not None and workflow_run is not None:
+ raise ValueError("task and workflow_run are mutually exclusive")
+
+ if task is not None:
+ tasks = [task]
+ organization_id = task.organization_id
+ workflow_run_id = task.workflow_run_id
+ run_status = str(task.status)
+ output_is_none = task.extracted_information is None
+ else:
+ assert workflow_run is not None
+ tasks = await app.DATABASE.tasks.get_tasks_by_workflow_run_id(workflow_run_id=workflow_run.workflow_run_id)
+ organization_id = workflow_run.organization_id
+ workflow_run_id = workflow_run.workflow_run_id
+ run_status = str(workflow_run.status)
+ workflow_outputs = await app.DATABASE.workflow_runs.get_workflow_run_output_parameters(
+ workflow_run_id=workflow_run.workflow_run_id
+ )
+ output_is_none = not workflow_outputs
+
+ steps: list[Step] = []
+ actions: list[Action] = []
+ for item in tasks:
+ task_steps = await app.DATABASE.tasks.get_task_steps(item.task_id, item.organization_id)
+ steps.extend(task_steps)
+ actions.extend(
+ await app.DATABASE.tasks.get_task_actions_hydrated(
+ item.task_id,
+ organization_id=item.organization_id,
+ )
+ )
+
+ if any(item.task_id == last_step.task_id for item in tasks):
+ steps = _merge_last_step(steps, last_step)
+ else:
+ steps.sort(key=lambda step: (step.created_at, step.order, step.retry_index))
+
+ return _LoadedContext(
+ tasks=tasks,
+ steps=steps,
+ actions=actions,
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ run_status=run_status,
+ output_is_none=output_is_none,
+ )
+
+
+def _preferred_page_artifact(artifacts: list[Artifact]) -> Artifact | None:
+ for artifact_type in _PAGE_ARTIFACT_TYPES:
+ matching = [artifact for artifact in artifacts if artifact.artifact_type == artifact_type]
+ if matching:
+ return max(matching, key=lambda artifact: artifact.created_at)
+ return None
+
+
+async def _retrieve_page_text(step: Step) -> str | None:
+ try:
+ artifacts = await app.DATABASE.artifacts.get_artifacts_for_task_step(
+ task_id=step.task_id,
+ step_id=step.step_id,
+ organization_id=step.organization_id,
+ )
+ artifact = _preferred_page_artifact(artifacts)
+ if artifact is None:
+ return None
+ content = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)
+ if not content or content.startswith(_ZIP_HEADER):
+ return None
+ return content.decode("utf-8")
+ except Exception: # noqa: BLE001 - page confirmation is optional shadow evidence
+ return None
+
+
+def _step_url(step: Step | None) -> str | None:
+ if step is None or step.output is None or step.output.browser_metadata is None:
+ return None
+ return step.output.browser_metadata.website_url
+
+
+def _evaluate_submission_signal(
+ *,
+ har_data: bytes,
+ candidate_windows: list[CandidateWindow],
+ candidate_step_pairs: list[tuple[Step, Step | None]],
+ detected_candidate_step_ids: list[str],
+ latest_candidate_step: Step | None,
+ pre_page_text: str | None,
+ post_page_text: str | None,
+ submit_intent_detected: bool,
+ browser_path: BrowserPath,
+ cua_run: bool,
+ coordinate_click: bool,
+) -> SubmissionVerdict:
+ har_too_large = len(har_data) > _MAX_HAR_BYTES
+ tier_a = (
+ TierAEvaluation(evidence=[], har_present=True, har_parsed=False, har_entry_count=0)
+ if har_too_large
+ else evaluate_tier_a(har_data, candidate_windows)
+ )
+ candidate_evaluations = [
+ CandidateEvaluation(
+ step_id=candidate_step.step_id,
+ tier_a=[evidence for evidence in tier_a.evidence if evidence.correlated_step_id == candidate_step.step_id],
+ tier_b=evaluate_tier_b(
+ pre_url=_step_url(candidate_step),
+ post_url=_step_url(candidate_post_step),
+ action_results=_flatten_action_results([candidate_step]),
+ pre_page_text=pre_page_text if candidate_step == latest_candidate_step else None,
+ post_page_text=post_page_text if candidate_step == latest_candidate_step else None,
+ ),
+ is_latest=candidate_step == latest_candidate_step,
+ )
+ for candidate_step, candidate_post_step in candidate_step_pairs
+ ]
+ verdict = combine(
+ tier_a=tier_a,
+ candidate_evaluations=candidate_evaluations,
+ detected_candidate_step_ids=detected_candidate_step_ids,
+ submit_intent_detected=submit_intent_detected,
+ browser_path=browser_path,
+ cua_run=cua_run,
+ coordinate_click=coordinate_click,
+ )
+ if har_too_large:
+ verdict.notes.append("har_too_large")
+ return verdict
+
+
+def _task_for_candidate_step(tasks: list[Task], candidate_step: Step | None) -> Task:
+ if candidate_step is not None:
+ for task in tasks:
+ if task.task_id == candidate_step.task_id:
+ return task
+ if not tasks:
+ raise ValueError("no tasks available for submission shadow")
+ return tasks[-1]
+
+
+async def run_submission_signal_shadow(
+ *,
+ har_data: bytes,
+ browser_state: BrowserState,
+ last_step: Step,
+ task: Task | None = None,
+ workflow_run: WorkflowRun | None = None,
+ browser_session_id: str | None = None,
+) -> None:
+ started_at = time.monotonic()
+ stage = "load_tasks"
+ if task is not None:
+ organization_id = task.organization_id
+ event_task_id = task.task_id
+ event_workflow_run_id = task.workflow_run_id
+ elif workflow_run is not None:
+ organization_id = workflow_run.organization_id
+ event_task_id = last_step.task_id
+ event_workflow_run_id = workflow_run.workflow_run_id
+ else:
+ organization_id = None
+ event_task_id = last_step.task_id
+ event_workflow_run_id = None
+ try:
+ context = await _load_context(task=task, workflow_run=workflow_run, last_step=last_step)
+ stage = "detect_candidates"
+ detection = detect_submit_candidates(context.actions)
+ candidate_step_pairs = find_candidate_step_pairs(detection.candidates, context.steps)
+ pre_step, post_step = candidate_step_pairs[-1] if candidate_step_pairs else (None, None)
+ context_task = _task_for_candidate_step(context.tasks, pre_step)
+
+ stage = "load_run"
+ cua_run = await service_utils.is_cua_task(task=context_task)
+ stage = "classify_browser_path"
+ artifacts = browser_state.browser_artifacts
+ browser_path = classify_browser_path(
+ browser_session_id=browser_session_id or (workflow_run.browser_session_id if workflow_run else None),
+ task_browser_session_id=context_task.browser_session_id,
+ remote_browser_session_id=artifacts.remote_browser_session_id,
+ task_browser_address=context_task.browser_address
+ or (workflow_run.browser_address if workflow_run is not None else None),
+ needs_cdp_frame_publisher=artifacts.needs_cdp_frame_publisher,
+ browser_type=settings.BROWSER_TYPE,
+ )
+
+ candidate_windows = build_candidate_windows(detection.candidates, context.steps)
+
+ stage = "retrieve_page_text"
+ pre_page_text = await _retrieve_page_text(pre_step) if pre_step is not None else None
+ post_page_text = await _retrieve_page_text(post_step) if post_step is not None else None
+
+ stage = "evaluate"
+ verdict = await asyncio.to_thread(
+ _evaluate_submission_signal,
+ har_data=har_data,
+ candidate_windows=candidate_windows,
+ candidate_step_pairs=candidate_step_pairs,
+ detected_candidate_step_ids=sorted({candidate.step_id for candidate in detection.candidates}),
+ latest_candidate_step=pre_step,
+ pre_page_text=pre_page_text,
+ post_page_text=post_page_text,
+ submit_intent_detected=detection.submit_intent_detected,
+ browser_path=browser_path,
+ cua_run=cua_run,
+ coordinate_click=detection.coordinate_click,
+ )
+ winning_step = next(
+ (
+ candidate_step
+ for candidate_step, _ in candidate_step_pairs
+ if candidate_step.step_id == verdict.winning_step_id
+ ),
+ None,
+ )
+ event_task = _task_for_candidate_step(context.tasks, winning_step or pre_step)
+
+ LOG.info(
+ _SHADOW_EVENT,
+ status="ok",
+ signal=verdict.signal.value,
+ tier_a_hit=bool(verdict.tier_a),
+ tier_b_hits=len(verdict.tier_b),
+ submit_intent_detected=verdict.submit_intent_detected,
+ run_status=context.run_status,
+ output_is_none=context.output_is_none,
+ har_present=verdict.har_present,
+ har_parsed=verdict.har_parsed,
+ har_entry_count=verdict.har_entry_count,
+ browser_path=verdict.browser_path.value,
+ capped=verdict.capped,
+ notes=verdict.notes,
+ duration_ms=_elapsed_ms(started_at),
+ task_id=event_task.task_id,
+ workflow_run_id=context.workflow_run_id,
+ organization_id=context.organization_id,
+ )
+ except Exception as exc: # noqa: BLE001 - shadow observability must never affect cleanup
+ LOG.warning(
+ _SHADOW_EVENT,
+ status="error",
+ error_stage=stage,
+ error_type=type(exc).__name__,
+ duration_ms=_elapsed_ms(started_at),
+ task_id=event_task_id,
+ workflow_run_id=event_workflow_run_id,
+ organization_id=organization_id,
+ )
+
+
+def schedule_submission_signal_shadow(
+ *,
+ har_data: bytes,
+ browser_state: BrowserState,
+ last_step: Step,
+ task: Task | None = None,
+ workflow_run: WorkflowRun | None = None,
+ browser_session_id: str | None = None,
+) -> asyncio.Task[None] | None:
+ if not settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW:
+ return None
+
+ try:
+ _prune_pending()
+ if len(_PENDING_SUBMISSION_SHADOW_TASKS) >= _MAX_PENDING:
+ LOG.warning("submission_shadow_task_cap_reached", pending=len(_PENDING_SUBMISSION_SHADOW_TASKS))
+ LOG.info(_SHADOW_EVENT, status="skipped", reason="pending_cap")
+ return None
+
+ runner = run_submission_signal_shadow(
+ har_data=har_data,
+ browser_state=browser_state,
+ last_step=last_step,
+ task=task,
+ workflow_run=workflow_run,
+ browser_session_id=browser_session_id,
+ )
+ try:
+ scheduled = asyncio.create_task(runner)
+ except Exception:
+ runner.close()
+ raise
+ try:
+ return _track(scheduled)
+ except Exception:
+ _PENDING_SUBMISSION_SHADOW_TASKS.discard(scheduled)
+ scheduled.cancel()
+ raise
+ except Exception as exc: # noqa: BLE001 - scheduler must never affect artifact persistence
+ try:
+ LOG.debug("submission_shadow_schedule_failed", error_type=type(exc).__name__)
+ except Exception: # noqa: BLE001 - logging must not break the no-throw boundary
+ pass
+ return None
diff --git a/skyvern/forge/sdk/submission/verifier.py b/skyvern/forge/sdk/submission/verifier.py
new file mode 100644
index 000000000..b8da1ca13
--- /dev/null
+++ b/skyvern/forge/sdk/submission/verifier.py
@@ -0,0 +1,473 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from pathlib import PurePosixPath
+from urllib.parse import urlsplit
+
+from skyvern.forge.sdk.copilot.completion_output_grounding import _boundary_delimited_present
+from skyvern.forge.sdk.models import Step
+from skyvern.forge.sdk.submission.models import (
+ BrowserPath,
+ CandidateWindow,
+ DownloadEvidence,
+ NetworkEvidence,
+ PageConfirmationEvidence,
+ SubmissionSignal,
+ SubmissionVerdict,
+ SubmitCandidate,
+ SubmitCandidateDetection,
+ TierAEvaluation,
+ TierBEvaluation,
+ TierBEvidence,
+ UrlTransitionEvidence,
+)
+from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import Action, ClickAction
+from skyvern.webeye.actions.responses import ActionResult
+
+_SUBMIT_VERBS = ("place order", "register", "complete", "confirm", "finish", "submit", "apply", "send")
+_CONFIRMATION_PHRASES = (
+ "confirmation number",
+ "successfully submitted",
+ "application received",
+ "submission received",
+ "we have received",
+ "thank you",
+)
+_NETWORK_METHODS = {"POST", "PUT", "PATCH"}
+_NETWORK_RESOURCE_TYPES = {"document", "xhr", "fetch"}
+_STATIC_MIME_PREFIXES = ("image/", "font/", "text/css")
+_STATIC_EXTENSIONS = {
+ ".avif",
+ ".bmp",
+ ".css",
+ ".eot",
+ ".gif",
+ ".ico",
+ ".jpeg",
+ ".jpg",
+ ".js",
+ ".map",
+ ".otf",
+ ".png",
+ ".svg",
+ ".ttf",
+ ".webp",
+ ".woff",
+ ".woff2",
+}
+_STEP_WINDOW_SKEW = timedelta(seconds=2)
+_STEP_WINDOW_TRAILING = timedelta(seconds=5)
+_CONFIRMATION_VALUE_RE = re.compile(r"^[\s:;#-]*([^\s<>,;]+)")
+
+
+@dataclass(frozen=True)
+class CandidateEvaluation:
+ step_id: str
+ tier_a: Sequence[NetworkEvidence]
+ tier_b: TierBEvaluation
+ is_latest: bool = False
+
+
+def classify_browser_path(
+ *,
+ browser_session_id: str | None,
+ task_browser_session_id: str | None,
+ remote_browser_session_id: str | None,
+ task_browser_address: str | None,
+ needs_cdp_frame_publisher: bool,
+ browser_type: str,
+) -> BrowserPath:
+ if browser_session_id or task_browser_session_id:
+ return BrowserPath.SESSION_ATTACHED
+ if remote_browser_session_id:
+ return BrowserPath.VENDOR_REUSED
+ if task_browser_address or needs_cdp_frame_publisher or browser_type == "cdp-connect":
+ return BrowserPath.CDP_CONNECT
+ if browser_type in {"chromium-headless", "chromium-headful"}:
+ return BrowserPath.SKYVERN_CREATED
+ return BrowserPath.UNKNOWN
+
+
+def _mapping(value: object) -> Mapping[str, object] | None:
+ return value if isinstance(value, Mapping) else None
+
+
+def _submit_shaped(element_data: Mapping[str, object]) -> bool:
+ tag_name = element_data.get("tagName")
+ if not isinstance(tag_name, str):
+ return False
+ tag_name = tag_name.strip().lower()
+ attributes = _mapping(element_data.get("attributes")) or {}
+ input_type = attributes.get("type")
+ if tag_name in {"button", "input"} and isinstance(input_type, str) and input_type.lower() == "submit":
+ return True
+ if tag_name not in {"button", "input", "a"}:
+ return False
+
+ text_values = [element_data.get("text"), attributes.get("value"), attributes.get("aria-label")]
+ return any(
+ _boundary_delimited_present(verb, value.strip().lower())
+ for value in text_values
+ if isinstance(value, str)
+ for verb in _SUBMIT_VERBS
+ )
+
+
+def detect_submit_candidates(actions: Sequence[Action]) -> SubmitCandidateDetection:
+ candidates: list[SubmitCandidate] = []
+ coordinate_click = False
+ for action in actions:
+ if action.action_type != ActionType.CLICK:
+ continue
+ if isinstance(action, ClickAction) and (action.x is not None or action.y is not None):
+ coordinate_click = True
+ continue
+ element_data = _mapping(action.skyvern_element_data)
+ if element_data is not None and _submit_shaped(element_data):
+ if action.step_id is None:
+ continue
+ candidates.append(
+ SubmitCandidate(
+ action_id=action.action_id,
+ task_id=action.task_id,
+ step_id=action.step_id,
+ )
+ )
+ return SubmitCandidateDetection(candidates=candidates, coordinate_click=coordinate_click)
+
+
+def _candidate_actions(
+ detection_or_candidates: SubmitCandidateDetection | Sequence[SubmitCandidate],
+) -> Sequence[SubmitCandidate]:
+ if isinstance(detection_or_candidates, SubmitCandidateDetection):
+ return detection_or_candidates.candidates
+ return detection_or_candidates
+
+
+def build_candidate_windows(
+ detection_or_candidates: SubmitCandidateDetection | Sequence[SubmitCandidate],
+ steps: Sequence[Step],
+) -> list[CandidateWindow]:
+ step_by_id = {step.step_id: step for step in steps}
+ windows: list[CandidateWindow] = []
+ seen_steps: set[str] = set()
+ for candidate in _candidate_actions(detection_or_candidates):
+ if candidate.step_id is None or candidate.step_id in seen_steps:
+ continue
+ step = step_by_id.get(candidate.step_id)
+ if step is None or (candidate.task_id is not None and candidate.task_id != step.task_id):
+ continue
+ seen_steps.add(step.step_id)
+ windows.append(
+ CandidateWindow(
+ task_id=step.task_id,
+ step_id=step.step_id,
+ candidate_action_id=candidate.action_id,
+ started_at=step.created_at - _STEP_WINDOW_SKEW,
+ ended_at=step.modified_at + _STEP_WINDOW_TRAILING,
+ )
+ )
+ return sorted(windows, key=lambda window: window.started_at)
+
+
+def find_candidate_step_pairs(
+ detection_or_candidates: SubmitCandidateDetection | Sequence[SubmitCandidate],
+ steps: Sequence[Step],
+) -> list[tuple[Step, Step | None]]:
+ ordered = sorted(steps, key=lambda step: (step.task_id, step.order, step.retry_index, step.created_at))
+ positions = {step.step_id: index for index, step in enumerate(ordered)}
+ pairs: list[tuple[Step, Step | None]] = []
+ seen_steps: set[str] = set()
+ for candidate in _candidate_actions(detection_or_candidates):
+ if candidate.step_id is None or candidate.step_id in seen_steps or candidate.step_id not in positions:
+ continue
+ pre_step = ordered[positions[candidate.step_id]]
+ if candidate.task_id is not None and candidate.task_id != pre_step.task_id:
+ continue
+ seen_steps.add(candidate.step_id)
+ index = positions[pre_step.step_id]
+ post_step = ordered[index + 1] if index + 1 < len(ordered) else None
+ if post_step is not None and post_step.task_id != pre_step.task_id:
+ post_step = None
+ pairs.append((pre_step, post_step))
+ return sorted(pairs, key=lambda pair: pair[0].modified_at)
+
+
+def find_latest_candidate_step_pair(
+ detection_or_candidates: SubmitCandidateDetection | Sequence[SubmitCandidate],
+ steps: Sequence[Step],
+) -> tuple[Step | None, Step | None]:
+ pairs = find_candidate_step_pairs(detection_or_candidates, steps)
+ if not pairs:
+ return None, None
+ return pairs[-1]
+
+
+def _utc(value: datetime) -> datetime:
+ if value.tzinfo is None:
+ return value.replace(tzinfo=UTC)
+ return value.astimezone(UTC)
+
+
+def _parse_har_time(value: str) -> datetime | None:
+ try:
+ return _utc(datetime.fromisoformat(value.replace("Z", "+00:00")))
+ except ValueError:
+ return None
+
+
+def _origin(url: str) -> str | None:
+ try:
+ parsed = urlsplit(url)
+ if not parsed.scheme or parsed.hostname is None:
+ return None
+ port = f":{parsed.port}" if parsed.port is not None else ""
+ return f"{parsed.scheme.lower()}://{parsed.hostname.lower()}{port}"
+ except ValueError:
+ return None
+
+
+def _path(url: str) -> str | None:
+ try:
+ parsed = urlsplit(url)
+ except ValueError:
+ return None
+ if not parsed.scheme or parsed.hostname is None:
+ return None
+ return parsed.path or "/"
+
+
+def _content_type(response: Mapping[str, object]) -> str:
+ content = _mapping(response.get("content"))
+ mime_type = content.get("mimeType") if content is not None else None
+ if isinstance(mime_type, str):
+ return mime_type.lower()
+ headers = response.get("headers")
+ if isinstance(headers, list):
+ for header in headers:
+ header_data = _mapping(header)
+ if header_data is None:
+ continue
+ name = header_data.get("name")
+ value = header_data.get("value")
+ if isinstance(name, str) and name.lower() == "content-type" and isinstance(value, str):
+ return value.lower()
+ return ""
+
+
+def _is_static(entry: Mapping[str, object], response: Mapping[str, object], url: str) -> bool:
+ resource_type = entry.get("_resourceType")
+ if isinstance(resource_type, str) and resource_type:
+ return resource_type.lower() not in _NETWORK_RESOURCE_TYPES
+ content_type = _content_type(response)
+ if any(content_type.startswith(prefix) for prefix in _STATIC_MIME_PREFIXES):
+ return True
+ return PurePosixPath(urlsplit(url).path.lower()).suffix in _STATIC_EXTENSIONS
+
+
+def evaluate_tier_a(har_bytes: bytes, candidate_windows: Sequence[CandidateWindow]) -> TierAEvaluation:
+ har_present = bool(har_bytes)
+ if not har_present:
+ return TierAEvaluation(evidence=[], har_present=False, har_parsed=False, har_entry_count=0)
+ try:
+ payload = json.loads(har_bytes)
+ log = _mapping(payload.get("log")) if isinstance(payload, Mapping) else None
+ entries = log.get("entries") if log is not None else None
+ except (UnicodeDecodeError, json.JSONDecodeError, RecursionError):
+ entries = None
+ if not isinstance(entries, list):
+ return TierAEvaluation(evidence=[], har_present=True, har_parsed=False, har_entry_count=0)
+
+ evidence: list[NetworkEvidence] = []
+ ambiguous_entry_count = 0
+ for entry_value in entries:
+ entry = _mapping(entry_value)
+ request = _mapping(entry.get("request")) if entry is not None else None
+ response = _mapping(entry.get("response")) if entry is not None else None
+ started_value = entry.get("startedDateTime") if entry is not None else None
+ if entry is None or request is None or response is None or not isinstance(started_value, str):
+ return TierAEvaluation(evidence=[], har_present=True, har_parsed=False, har_entry_count=len(entries))
+ method = request.get("method")
+ url = request.get("url")
+ status = response.get("status")
+ started_at = _parse_har_time(started_value)
+ origin = _origin(url) if isinstance(url, str) else None
+ if (
+ not isinstance(method, str)
+ or not isinstance(url, str)
+ or not isinstance(status, int)
+ or isinstance(status, bool)
+ or started_at is None
+ or origin is None
+ ):
+ return TierAEvaluation(evidence=[], har_present=True, har_parsed=False, har_entry_count=len(entries))
+ if method.upper() not in _NETWORK_METHODS or not 200 <= status < 400 or _is_static(entry, response, url):
+ continue
+ matching_windows = [
+ window for window in candidate_windows if _utc(window.started_at) <= started_at <= _utc(window.ended_at)
+ ]
+ if len(matching_windows) > 1:
+ ambiguous_entry_count += 1
+ continue
+ if len(matching_windows) == 1:
+ evidence.append(
+ NetworkEvidence(
+ origin=origin,
+ method=method.upper(),
+ status=status,
+ started_at=started_at,
+ correlated_step_id=matching_windows[0].step_id,
+ )
+ )
+ return TierAEvaluation(
+ evidence=evidence,
+ har_present=True,
+ har_parsed=True,
+ har_entry_count=len(entries),
+ ambiguous_entry_count=ambiguous_entry_count,
+ )
+
+
+def _confirmation_value(phrase: str, normalized_post_text: str) -> str:
+ if phrase != "confirmation number":
+ return phrase
+ index = normalized_post_text.find(phrase)
+ match = _CONFIRMATION_VALUE_RE.match(normalized_post_text[index + len(phrase) :])
+ return match.group(1) if match else phrase
+
+
+def evaluate_tier_b(
+ *,
+ pre_url: str | None,
+ post_url: str | None,
+ action_results: Sequence[ActionResult],
+ pre_page_text: str | None,
+ post_page_text: str | None,
+) -> TierBEvaluation:
+ evidence: list[TierBEvidence] = []
+ if pre_url is not None and post_url is not None:
+ from_origin = _origin(pre_url)
+ to_origin = _origin(post_url)
+ from_path = _path(pre_url)
+ to_path = _path(post_url)
+ if from_origin is not None and to_origin is not None and from_path is not None and to_path is not None:
+ path_changed = from_path != to_path
+ if from_origin != to_origin or path_changed:
+ evidence.append(
+ UrlTransitionEvidence(
+ from_origin=from_origin,
+ to_origin=to_origin,
+ path_changed=path_changed,
+ )
+ )
+
+ file_count = sum(
+ max(len(result.downloaded_files or []), 1 if result.download_triggered else 0) for result in action_results
+ )
+ if file_count:
+ evidence.append(DownloadEvidence(file_count=file_count))
+
+ page_evaluated = False
+ if pre_page_text is not None and post_page_text is not None:
+ page_evaluated = True
+ normalized_pre = pre_page_text.casefold()
+ normalized_post = post_page_text.casefold()
+ for phrase in _CONFIRMATION_PHRASES:
+ if _boundary_delimited_present(phrase, normalized_post) and not _boundary_delimited_present(
+ phrase, normalized_pre
+ ):
+ matched_value = _confirmation_value(phrase, normalized_post)
+ evidence.append(
+ PageConfirmationEvidence(
+ phrase=phrase,
+ value_sha256=hashlib.sha256(matched_value.encode()).hexdigest(),
+ absent_pre_submit=True,
+ )
+ )
+ break
+ return TierBEvaluation(evidence=evidence, page_confirmation_evaluated=page_evaluated)
+
+
+def combine(
+ *,
+ tier_a: TierAEvaluation,
+ candidate_evaluations: Sequence[CandidateEvaluation],
+ detected_candidate_step_ids: Sequence[str],
+ submit_intent_detected: bool,
+ browser_path: BrowserPath,
+ cua_run: bool = False,
+ coordinate_click: bool = False,
+) -> SubmissionVerdict:
+ tier_a_allowed = browser_path == BrowserPath.SKYVERN_CREATED
+
+ def candidate_strength(candidate: CandidateEvaluation) -> int:
+ has_tier_a = tier_a_allowed and bool(candidate.tier_a)
+ has_tier_b = bool(candidate.tier_b.evidence)
+ if has_tier_a and has_tier_b:
+ return 2
+ return int(has_tier_a or has_tier_b)
+
+ winning_candidate = max(
+ candidate_evaluations,
+ key=lambda candidate: (candidate_strength(candidate), candidate.is_latest),
+ default=None,
+ )
+ tier_a_evidence = list(winning_candidate.tier_a) if winning_candidate is not None and tier_a_allowed else []
+ tier_b_evidence = list(winning_candidate.tier_b.evidence) if winning_candidate is not None else []
+ winning_strength = candidate_strength(winning_candidate) if winning_candidate is not None else 0
+ capped = False
+ notes: list[str] = []
+ if tier_a.ambiguous_entry_count:
+ notes.append(f"ambiguous_entries:{tier_a.ambiguous_entry_count}")
+
+ if cua_run or coordinate_click:
+ signal = SubmissionSignal.NOT_EVALUATED
+ notes.append("cua_or_coordinate_click")
+ elif not submit_intent_detected:
+ signal = SubmissionSignal.NOT_EVALUATED
+ notes.append("submit_intent_not_detected")
+ elif browser_path != BrowserPath.SKYVERN_CREATED:
+ capped = True
+ signal = SubmissionSignal.SUBMITTED_LIKELY if winning_strength else SubmissionSignal.NOT_EVALUATED
+ notes.append("tier_b_only_browser_path")
+ elif winning_strength == 2:
+ signal = SubmissionSignal.SUBMITTED_VERIFIED
+ elif winning_strength == 1:
+ signal = SubmissionSignal.SUBMITTED_LIKELY
+ elif (
+ tier_a.har_present
+ and tier_a.har_parsed
+ and tier_a.har_entry_count > 0
+ # An ambiguous entry is a discarded, possibly real submission POST — absence is not affirmative.
+ and tier_a.ambiguous_entry_count == 0
+ and len(candidate_evaluations) > 0
+ # A detected candidate whose step could not be mapped (actions.step_id has no FK) is unknown
+ # evidence — evaluations must cover every detected candidate step.
+ and set(detected_candidate_step_ids) <= {candidate.step_id for candidate in candidate_evaluations}
+ # Every candidate's page-confirmation check must have actually run, not just the latest one's.
+ and all(candidate.tier_b.page_confirmation_evaluated for candidate in candidate_evaluations)
+ and all(not candidate.tier_a and not candidate.tier_b.evidence for candidate in candidate_evaluations)
+ ):
+ signal = SubmissionSignal.NOT_SUBMITTED
+ else:
+ signal = SubmissionSignal.NOT_EVALUATED
+
+ return SubmissionVerdict(
+ signal=signal,
+ tier_a=tier_a_evidence,
+ tier_b=tier_b_evidence,
+ submit_intent_detected=submit_intent_detected,
+ har_present=tier_a.har_present,
+ har_parsed=tier_a.har_parsed,
+ har_entry_count=tier_a.har_entry_count,
+ browser_path=browser_path,
+ winning_step_id=winning_candidate.step_id if winning_candidate is not None else None,
+ capped=capped,
+ notes=notes,
+ )
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 5e2512174..9adaf141c 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -76,6 +76,7 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import (
)
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock, WorkflowRunTimeline, WorkflowRunTimelineType
+from skyvern.forge.sdk.submission import shadow as submission_shadow
from skyvern.forge.sdk.trace import traced
from skyvern.forge.sdk.workflow.browser_profile_key import (
build_browser_profile_key_digest,
@@ -7421,6 +7422,13 @@ class WorkflowService:
workflow_run_id=workflow_run.workflow_run_id,
browser_state=browser_state,
)
+ if settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW:
+ submission_shadow.schedule_submission_signal_shadow(
+ har_data=har_data,
+ browser_state=browser_state,
+ last_step=last_step,
+ workflow_run=workflow_run,
+ )
LOG.debug("Persisting har data", har_size=len(har_data))
if har_data:
await app.ARTIFACT_MANAGER.create_artifact(
@@ -7503,6 +7511,13 @@ class WorkflowService:
workflow_run_id=workflow_run.workflow_run_id,
browser_state=browser_state,
)
+ if settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW:
+ submission_shadow.schedule_submission_signal_shadow(
+ har_data=har_data,
+ browser_state=browser_state,
+ last_step=last_step,
+ workflow_run=workflow_run,
+ )
LOG.debug("Persisting har data (bundled)", har_size=len(har_data))
if har_data:
task_archive_entries["har.har"] = (ArtifactType.HAR, har_data)
diff --git a/skyvern/services/service_utils.py b/skyvern/services/service_utils.py
index d97da8b32..fc845eb72 100644
--- a/skyvern/services/service_utils.py
+++ b/skyvern/services/service_utils.py
@@ -1,4 +1,5 @@
from skyvern.forge import app
+from skyvern.forge.sdk.db.exceptions import NotFoundError
from skyvern.forge.sdk.schemas.tasks import Task
from skyvern.schemas.runs import CUA_ENGINES, CUA_RUN_TYPES
@@ -11,11 +12,15 @@ async def is_cua_task(
if task.workflow_run_id:
# it's a task based block, should look up the block run to see if it's a CUA task
- block = await app.DATABASE.observer.get_workflow_run_block_by_task_id(
- task_id=task.task_id,
- organization_id=task.organization_id,
- )
- if block.engine is not None and block.engine in CUA_ENGINES:
+ try:
+ block = await app.DATABASE.observer.get_workflow_run_block_by_task_id(
+ task_id=task.task_id,
+ organization_id=task.organization_id,
+ )
+ except NotFoundError:
+ # a missing block row means "unknown", not CUA — fall through to the run-type check
+ block = None
+ if block is not None and block.engine is not None and block.engine in CUA_ENGINES:
return True
run = await app.DATABASE.tasks.get_run(
diff --git a/tests/unit/submission/test_shadow.py b/tests/unit/submission/test_shadow.py
new file mode 100644
index 000000000..8c5dc903c
--- /dev/null
+++ b/tests/unit/submission/test_shadow.py
@@ -0,0 +1,774 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from typing import Any, cast
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from structlog.testing import capture_logs
+
+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType
+from skyvern.forge.sdk.models import Step, StepStatus
+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
+from skyvern.forge.sdk.submission import shadow
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
+from skyvern.schemas.run_enums import RunEngine, RunType
+from skyvern.schemas.steps import AgentStepOutput, BrowserMetadata
+from skyvern.webeye.actions.actions import ClickAction
+from skyvern.webeye.actions.responses import ActionResult
+from skyvern.webeye.browser_artifacts import BrowserArtifacts
+from skyvern.webeye.browser_state import BrowserState
+
+NOW = datetime(2026, 7, 9, 12, 0, 0)
+
+
+def _task(task_id: str = "task-1", *, workflow_run_id: str | None = None) -> Task:
+ return Task(
+ task_id=task_id,
+ status=TaskStatus.completed,
+ created_at=NOW,
+ modified_at=NOW,
+ organization_id="org-1",
+ workflow_run_id=workflow_run_id,
+ url="https://example.com/form",
+ )
+
+
+def _submit_action(*, task_id: str = "task-1", step_id: str = "step-1") -> ClickAction:
+ return ClickAction(
+ element_id="submit-button",
+ task_id=task_id,
+ step_id=step_id,
+ created_at=NOW + timedelta(seconds=2),
+ skyvern_element_data={
+ "tagName": "button",
+ "attributes": {"type": "submit"},
+ "text": "Submit",
+ },
+ )
+
+
+def _step(
+ step_id: str,
+ order: int,
+ *,
+ task_id: str = "task-1",
+ url: str = "https://example.com/form",
+ action: ClickAction | None = None,
+) -> Step:
+ pairs = [] if action is None else [(action, [ActionResult(success=True)])]
+ return Step(
+ created_at=NOW + timedelta(seconds=order * 10),
+ modified_at=NOW + timedelta(seconds=order * 10 + 5),
+ task_id=task_id,
+ step_id=step_id,
+ status=StepStatus.completed,
+ output=AgentStepOutput(
+ actions_and_results=pairs,
+ browser_metadata=BrowserMetadata(website_url=url),
+ ),
+ order=order,
+ is_last=False,
+ organization_id="org-1",
+ )
+
+
+def _workflow_run() -> WorkflowRun:
+ return WorkflowRun(
+ workflow_run_id="workflow-run-1",
+ workflow_id="workflow-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ status=WorkflowRunStatus.completed,
+ created_at=NOW,
+ modified_at=NOW,
+ )
+
+
+def _browser_state() -> BrowserState:
+ return cast(
+ BrowserState,
+ SimpleNamespace(
+ browser_artifacts=BrowserArtifacts(),
+ ),
+ )
+
+
+def _har(*, method: str = "POST", status: int = 200, offset_seconds: int = 12) -> bytes:
+ return json.dumps(
+ {
+ "log": {
+ "entries": [
+ {
+ "startedDateTime": (NOW + timedelta(seconds=offset_seconds)).replace(tzinfo=UTC).isoformat(),
+ "request": {"method": method, "url": "https://example.com/submit?token=private"},
+ "response": {"status": status, "content": {"mimeType": "application/json"}},
+ "_resourceType": "xhr",
+ }
+ ]
+ }
+ }
+ ).encode()
+
+
+def _artifact(artifact_id: str, step_id: str, artifact_type: ArtifactType) -> Artifact:
+ return Artifact(
+ artifact_id=artifact_id,
+ artifact_type=artifact_type,
+ uri=f"s3://artifacts/{artifact_id}",
+ task_id="task-1",
+ step_id=step_id,
+ organization_id="org-1",
+ created_at=NOW,
+ modified_at=NOW,
+ )
+
+
+def _install_task_dependencies(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ steps: list[Step],
+ actions: list[ClickAction],
+ artifact_payloads: dict[str, bytes | None],
+ include_visible_artifacts: bool = True,
+) -> tuple[SimpleNamespace, AsyncMock]:
+ visible_by_step = {
+ step.step_id: _artifact(f"visible-{step.step_id}", step.step_id, ArtifactType.VISIBLE_ELEMENTS_TREE_IN_PROMPT)
+ for step in steps
+ }
+ html_by_step = {
+ step.step_id: _artifact(f"html-{step.step_id}", step.step_id, ArtifactType.HTML_SCRAPE) for step in steps
+ }
+
+ async def get_artifacts_for_task_step(
+ task_id: str,
+ step_id: str,
+ organization_id: str | None = None,
+ ) -> list[Artifact]:
+ assert task_id == "task-1"
+ assert organization_id == "org-1"
+ if include_visible_artifacts:
+ return [html_by_step[step_id], visible_by_step[step_id]]
+ return [html_by_step[step_id]]
+
+ tasks_repo = SimpleNamespace(
+ get_task_steps=AsyncMock(return_value=steps),
+ get_task_actions_hydrated=AsyncMock(return_value=actions),
+ get_run=AsyncMock(return_value=SimpleNamespace(task_run_type=RunType.task_v1)),
+ )
+ artifacts_repo = SimpleNamespace(get_artifacts_for_task_step=AsyncMock(side_effect=get_artifacts_for_task_step))
+ observer_repo = SimpleNamespace(get_workflow_run_block_by_task_id=AsyncMock())
+ database = SimpleNamespace(tasks=tasks_repo, artifacts=artifacts_repo, observer=observer_repo)
+ retrieve_artifact = AsyncMock(side_effect=lambda artifact: artifact_payloads.get(artifact.artifact_id))
+ monkeypatch.setattr(shadow.app, "DATABASE", database)
+ monkeypatch.setattr(shadow.app, "ARTIFACT_MANAGER", SimpleNamespace(retrieve_artifact=retrieve_artifact))
+ return database, retrieve_artifact
+
+
[email protected](autouse=True)
+def _clear_pending_tasks() -> None:
+ shadow._PENDING_SUBMISSION_SHADOW_TASKS.clear()
+ yield
+ shadow._PENDING_SUBMISSION_SHADOW_TASKS.clear()
+
+
+def test_flag_off_returns_before_pruning_or_logging(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", False)
+ monkeypatch.setattr(shadow, "_prune_pending", Mock(side_effect=AssertionError("must not prune")))
+ task = _task()
+ last_step = _step("step-1", 1)
+
+ with capture_logs() as logs:
+ scheduled = shadow.schedule_submission_signal_shadow(
+ har_data=b"",
+ browser_state=_browser_state(),
+ last_step=last_step,
+ task=task,
+ )
+
+ assert scheduled is None
+ assert shadow._PENDING_SUBMISSION_SHADOW_TASKS == set()
+ assert logs == []
+
+
+def test_pending_task_cap_drops_work_and_warns(monkeypatch: pytest.MonkeyPatch) -> None:
+ class _PendingMarker:
+ def done(self) -> bool:
+ return False
+
+ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", True)
+ assert shadow._MAX_PENDING == 20
+ pending: set[Any] = {_PendingMarker() for _ in range(shadow._MAX_PENDING)}
+ monkeypatch.setattr(shadow, "_PENDING_SUBMISSION_SHADOW_TASKS", pending)
+
+ with capture_logs() as logs:
+ scheduled = shadow.schedule_submission_signal_shadow(
+ har_data=b"",
+ browser_state=_browser_state(),
+ last_step=_step("step-1", 1),
+ task=_task(),
+ )
+
+ assert scheduled is None
+ assert len(shadow._PENDING_SUBMISSION_SHADOW_TASKS) == shadow._MAX_PENDING
+ assert logs == [
+ {
+ "event": "submission_shadow_task_cap_reached",
+ "log_level": "warning",
+ "pending": shadow._MAX_PENDING,
+ },
+ {
+ "event": "submission_signal_shadow",
+ "log_level": "info",
+ "reason": "pending_cap",
+ "status": "skipped",
+ },
+ ]
+
+
+def test_scheduler_swallows_setup_failures(monkeypatch: pytest.MonkeyPatch) -> None:
+ log = SimpleNamespace(debug=Mock())
+ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", True)
+ monkeypatch.setattr(shadow, "_prune_pending", Mock(side_effect=RuntimeError("private scheduler detail")))
+ monkeypatch.setattr(shadow, "LOG", log)
+
+ scheduled = shadow.schedule_submission_signal_shadow(
+ har_data=b"",
+ browser_state=_browser_state(),
+ last_step=_step("step-1", 1),
+ task=_task(),
+ )
+
+ assert scheduled is None
+ log.debug.assert_called_once_with("submission_shadow_schedule_failed", error_type="RuntimeError")
+
+
+async def test_scheduler_prunes_done_tasks_before_applying_cap(monkeypatch: pytest.MonkeyPatch) -> None:
+ class _DoneMarker:
+ def done(self) -> bool:
+ return True
+
+ async def fake_runner(**_kwargs: object) -> None:
+ return None
+
+ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", True)
+ monkeypatch.setattr(shadow, "run_submission_signal_shadow", fake_runner)
+ pending: set[Any] = {_DoneMarker() for _ in range(shadow._MAX_PENDING)}
+ monkeypatch.setattr(shadow, "_PENDING_SUBMISSION_SHADOW_TASKS", pending)
+
+ with capture_logs() as logs:
+ scheduled = shadow.schedule_submission_signal_shadow(
+ har_data=b"",
+ browser_state=_browser_state(),
+ last_step=_step("step-1", 1),
+ task=_task(),
+ )
+
+ assert scheduled is not None
+ await scheduled
+ await asyncio.sleep(0)
+ assert shadow._PENDING_SUBMISSION_SHADOW_TASKS == set()
+ assert logs == []
+
+
+async def test_scheduler_returns_and_tracks_background_task(monkeypatch: pytest.MonkeyPatch) -> None:
+ started = asyncio.Event()
+ release = asyncio.Event()
+
+ async def fake_runner(**_kwargs: object) -> None:
+ started.set()
+ await release.wait()
+
+ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", True)
+ monkeypatch.setattr(shadow, "run_submission_signal_shadow", fake_runner)
+
+ scheduled = shadow.schedule_submission_signal_shadow(
+ har_data=b"",
+ browser_state=_browser_state(),
+ last_step=_step("step-1", 1),
+ task=_task(),
+ )
+
+ assert scheduled is not None
+ await started.wait()
+ assert scheduled in shadow._PENDING_SUBMISSION_SHADOW_TASKS
+ release.set()
+ await scheduled
+ await asyncio.sleep(0)
+ assert scheduled not in shadow._PENDING_SUBMISSION_SHADOW_TASKS
+
+
+ ("include_visible_artifacts", "artifact_prefix"),
+ [(True, "visible"), (False, "html")],
+)
+async def test_runner_uses_hydrated_actions_next_step_text_and_emits_one_aggregate_event(
+ monkeypatch: pytest.MonkeyPatch,
+ include_visible_artifacts: bool,
+ artifact_prefix: str,
+) -> None:
+ action = _submit_action()
+ pre_step = _step("step-1", 1, action=action)
+ post_step = _step("step-2", 2)
+ database, retrieve_artifact = _install_task_dependencies(
+ monkeypatch,
+ steps=[pre_step, post_step],
+ actions=[action],
+ artifact_payloads={
+ "visible-step-1": b"Review the details before sending.",
+ "visible-step-2": b"Thank you. Your request is complete.",
+ "html-step-1": b"Review the details before sending.",
+ "html-step-2": b"Thank you. Your request is complete.",
+ },
+ include_visible_artifacts=include_visible_artifacts,
+ )
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+ to_thread = AsyncMock(side_effect=lambda function, *args, **kwargs: function(*args, **kwargs))
+ monkeypatch.setattr(shadow.asyncio, "to_thread", to_thread)
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(),
+ browser_state=_browser_state(),
+ last_step=post_step,
+ task=_task(),
+ )
+
+ events = [record for record in logs if record["event"] == "submission_signal_shadow"]
+ assert len(events) == 1
+ assert events[0]["status"] == "ok"
+ assert events[0]["signal"] == "submitted_verified"
+ assert events[0]["tier_a_hit"] is True
+ assert events[0]["tier_b_hits"] >= 1
+ assert "evidence" not in events[0]
+ to_thread.assert_awaited_once()
+ database.tasks.get_task_actions_hydrated.assert_awaited_once_with("task-1", organization_id="org-1")
+ database.tasks.get_run.assert_awaited_once_with(run_id="task-1", organization_id="org-1")
+ assert [call.args[0].artifact_id for call in retrieve_artifact.await_args_list] == [
+ f"{artifact_prefix}-step-1",
+ f"{artifact_prefix}-step-2",
+ ]
+
+
+async def test_runner_skips_oversized_har_without_allowing_not_submitted(monkeypatch: pytest.MonkeyPatch) -> None:
+ action = _submit_action()
+ pre_step = _step("step-1", 1, action=action)
+ post_step = _step("step-2", 2)
+ _install_task_dependencies(
+ monkeypatch,
+ steps=[pre_step, post_step],
+ actions=[action],
+ artifact_payloads={
+ "visible-step-1": b"Review the details before sending.",
+ "visible-step-2": b"The form remains ready.",
+ },
+ )
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+ monkeypatch.setattr(shadow, "_MAX_HAR_BYTES", 10)
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=b"x" * 11,
+ browser_state=_browser_state(),
+ last_step=post_step,
+ task=_task(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["signal"] == "not_evaluated"
+ assert event["har_present"] is True
+ assert event["har_parsed"] is False
+ assert "har_too_large" in event["notes"]
+
+
+async def test_unmapped_candidate_step_cannot_produce_not_submitted(monkeypatch: pytest.MonkeyPatch) -> None:
+ mapped_action = _submit_action()
+ ghost_action = _submit_action(step_id="step-ghost")
+ pre_step = _step("step-1", 1, action=mapped_action)
+ post_step = _step("step-2", 2)
+ _install_task_dependencies(
+ monkeypatch,
+ steps=[pre_step, post_step],
+ actions=[mapped_action, ghost_action],
+ artifact_payloads={
+ "visible-step-1": b"Review the details before sending.",
+ "visible-step-2": b"The form remains ready.",
+ },
+ )
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(method="GET"),
+ browser_state=_browser_state(),
+ last_step=post_step,
+ task=_task(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["har_present"] is True
+ assert event["har_parsed"] is True
+ assert event["signal"] == "not_evaluated"
+
+
+async def test_zip_page_artifacts_cannot_produce_not_submitted(monkeypatch: pytest.MonkeyPatch) -> None:
+ action = _submit_action()
+ pre_step = _step("step-1", 1, action=action)
+ post_step = _step("step-2", 2)
+ _, retrieve_artifact = _install_task_dependencies(
+ monkeypatch,
+ steps=[pre_step, post_step],
+ actions=[action],
+ artifact_payloads={
+ "visible-step-1": b"PK\x03\x04pre-archive",
+ "visible-step-2": b"PK\x03\x04post-archive",
+ },
+ )
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(method="GET"),
+ browser_state=_browser_state(),
+ last_step=post_step,
+ task=_task(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["status"] == "ok"
+ assert event["signal"] == "not_evaluated"
+ assert event["signal"] != "not_submitted"
+ assert retrieve_artifact.await_count == 2
+
+
+async def test_unrelated_step_download_does_not_count_as_candidate_evidence(monkeypatch: pytest.MonkeyPatch) -> None:
+ unrelated_step = _step("step-0", 0)
+ assert unrelated_step.output is not None
+ unrelated_step.output.action_results = [ActionResult(success=True, download_triggered=True)]
+ action = _submit_action()
+ pre_step = _step("step-1", 1, action=action)
+ post_step = _step("step-2", 2)
+ _install_task_dependencies(
+ monkeypatch,
+ steps=[unrelated_step, pre_step, post_step],
+ actions=[action],
+ artifact_payloads={
+ "visible-step-1": b"Review the details before sending.",
+ "visible-step-2": b"The form remains ready.",
+ },
+ )
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(method="GET"),
+ browser_state=_browser_state(),
+ last_step=post_step,
+ task=_task(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["status"] == "ok"
+ assert event["signal"] == "not_submitted"
+ assert event["tier_b_hits"] == 0
+
+
+async def test_latest_unpaired_candidate_cannot_reuse_older_page_completeness(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ older_action = _submit_action(step_id="step-1")
+ latest_action = _submit_action(step_id="step-3")
+ older_step = _step("step-1", 1, action=older_action)
+ older_post_step = _step("step-2", 2)
+ latest_step = _step("step-3", 3, action=latest_action)
+ _, retrieve_artifact = _install_task_dependencies(
+ monkeypatch,
+ steps=[older_step, older_post_step, latest_step],
+ actions=[older_action, latest_action],
+ artifact_payloads={
+ "visible-step-1": b"Review the details.",
+ "visible-step-2": b"The form remains ready.",
+ "visible-step-3": b"Review the latest details.",
+ },
+ )
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(method="GET"),
+ browser_state=_browser_state(),
+ last_step=latest_step,
+ task=_task(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["status"] == "ok"
+ assert event["signal"] == "not_evaluated"
+ assert [call.args[0].step_id for call in retrieve_artifact.await_args_list] == ["step-3"]
+
+
+async def test_runner_does_not_cross_pair_older_network_with_latest_transition(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ older_action = _submit_action(step_id="step-1")
+ latest_action = _submit_action(step_id="step-3")
+ older_step = _step("step-1", 1, action=older_action)
+ older_post_step = _step("step-2", 2)
+ latest_step = _step("step-3", 3, action=latest_action)
+ latest_post_step = _step("step-4", 4, url="https://example.com/complete")
+ _install_task_dependencies(
+ monkeypatch,
+ steps=[older_step, older_post_step, latest_step, latest_post_step],
+ actions=[older_action, latest_action],
+ artifact_payloads={
+ "visible-step-3": b"Review the latest details.",
+ "visible-step-4": b"The form remains ready.",
+ },
+ )
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(offset_seconds=12),
+ browser_state=_browser_state(),
+ last_step=latest_post_step,
+ task=_task(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["status"] == "ok"
+ assert event["signal"] == "submitted_likely"
+ assert event["tier_a_hit"] is False
+ assert event["tier_b_hits"] == 1
+
+
+async def test_runner_can_select_an_older_verified_candidate_without_extra_page_fetches(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ older_action = _submit_action(step_id="step-1")
+ latest_action = _submit_action(step_id="step-3")
+ older_step = _step("step-1", 1, action=older_action)
+ older_post_step = _step("step-2", 2, url="https://example.com/complete")
+ latest_step = _step("step-3", 3, action=latest_action)
+ _, retrieve_artifact = _install_task_dependencies(
+ monkeypatch,
+ steps=[older_step, older_post_step, latest_step],
+ actions=[older_action, latest_action],
+ artifact_payloads={"visible-step-3": b"Review the latest details."},
+ )
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(offset_seconds=12),
+ browser_state=_browser_state(),
+ last_step=latest_step,
+ task=_task(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["status"] == "ok"
+ assert event["signal"] == "submitted_verified"
+ assert event["tier_a_hit"] is True
+ assert event["tier_b_hits"] == 1
+ assert [call.args[0].step_id for call in retrieve_artifact.await_args_list] == ["step-3"]
+
+
+ ("run_type", "include_coordinate_click"),
+ [(RunType.openai_cua, False), (RunType.task_v1, True)],
+)
+async def test_cua_run_and_coordinate_click_overrides_are_applied_end_to_end(
+ monkeypatch: pytest.MonkeyPatch,
+ run_type: RunType,
+ include_coordinate_click: bool,
+) -> None:
+ action = _submit_action()
+ actions = [action]
+ if include_coordinate_click:
+ actions.append(ClickAction(element_id="", step_id="step-1", x=10, y=20))
+ pre_step = _step("step-1", 1, action=action)
+ post_step = _step("step-2", 2)
+ database, _ = _install_task_dependencies(
+ monkeypatch,
+ steps=[pre_step, post_step],
+ actions=actions,
+ artifact_payloads={
+ "visible-step-1": b"Review the details.",
+ "visible-step-2": b"Thank you. Your request is complete.",
+ },
+ )
+ database.tasks.get_run.return_value = SimpleNamespace(task_run_type=run_type)
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(),
+ browser_state=_browser_state(),
+ last_step=post_step,
+ task=_task(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["status"] == "ok"
+ assert event["signal"] == "not_evaluated"
+ database.tasks.get_run.assert_awaited_once_with(run_id="task-1", organization_id="org-1")
+
+
+async def test_workflow_uses_all_tasks_and_block_engine_for_cua_detection(monkeypatch: pytest.MonkeyPatch) -> None:
+ first_task = _task("task-a", workflow_run_id="workflow-run-1")
+ candidate_task = _task("task-1", workflow_run_id="workflow-run-1")
+ action = _submit_action()
+ first_step = _step("step-a", 1, task_id="task-a")
+ pre_step = _step("step-1", 2, action=action)
+ post_step = _step("step-2", 3)
+
+ async def get_steps(task_id: str, organization_id: str) -> list[Step]:
+ assert organization_id == "org-1"
+ return [first_step] if task_id == "task-a" else [pre_step, post_step]
+
+ async def get_actions(task_id: str, organization_id: str | None = None) -> list[ClickAction]:
+ assert organization_id == "org-1"
+ return [] if task_id == "task-a" else [action]
+
+ tasks_repo = SimpleNamespace(
+ get_tasks_by_workflow_run_id=AsyncMock(return_value=[first_task, candidate_task]),
+ get_task_steps=AsyncMock(side_effect=get_steps),
+ get_task_actions_hydrated=AsyncMock(side_effect=get_actions),
+ get_run=AsyncMock(return_value=SimpleNamespace(task_run_type=RunType.task_v1)),
+ )
+ artifacts_repo = SimpleNamespace(get_artifacts_for_task_step=AsyncMock(return_value=[]))
+ observer_repo = SimpleNamespace(
+ get_workflow_run_block_by_task_id=AsyncMock(return_value=SimpleNamespace(engine=RunEngine.openai_cua))
+ )
+ workflow_runs_repo = SimpleNamespace(
+ get_workflow_run_output_parameters=AsyncMock(return_value=[SimpleNamespace(value="workflow-output")])
+ )
+ monkeypatch.setattr(
+ shadow.app,
+ "DATABASE",
+ SimpleNamespace(
+ tasks=tasks_repo,
+ artifacts=artifacts_repo,
+ observer=observer_repo,
+ workflow_runs=workflow_runs_repo,
+ ),
+ )
+ monkeypatch.setattr(shadow.app, "ARTIFACT_MANAGER", SimpleNamespace(retrieve_artifact=AsyncMock()))
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(),
+ browser_state=_browser_state(),
+ last_step=post_step,
+ workflow_run=_workflow_run(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["status"] == "ok"
+ assert event["signal"] == "not_evaluated"
+ assert event["task_id"] == "task-1"
+ assert event["workflow_run_id"] == "workflow-run-1"
+ assert event["output_is_none"] is False
+ tasks_repo.get_tasks_by_workflow_run_id.assert_awaited_once_with(workflow_run_id="workflow-run-1")
+ assert tasks_repo.get_task_steps.await_count == 2
+ assert tasks_repo.get_task_actions_hydrated.await_count == 2
+ observer_repo.get_workflow_run_block_by_task_id.assert_awaited_once_with(
+ task_id="task-1",
+ organization_id="org-1",
+ )
+ workflow_runs_repo.get_workflow_run_output_parameters.assert_awaited_once_with(workflow_run_id="workflow-run-1")
+
+
+async def test_workflow_event_task_follows_the_winning_candidate(monkeypatch: pytest.MonkeyPatch) -> None:
+ older_task = _task("task-a", workflow_run_id="workflow-run-1")
+ latest_task = _task("task-b", workflow_run_id="workflow-run-1")
+ older_action = _submit_action(task_id="task-a", step_id="step-a1")
+ latest_action = _submit_action(task_id="task-b", step_id="step-b1")
+ older_step = _step("step-a1", 1, task_id="task-a", action=older_action)
+ older_post_step = _step("step-a2", 2, task_id="task-a", url="https://example.com/complete")
+ latest_step = _step("step-b1", 3, task_id="task-b", action=latest_action)
+
+ async def get_steps(task_id: str, organization_id: str) -> list[Step]:
+ assert organization_id == "org-1"
+ return [older_step, older_post_step] if task_id == "task-a" else [latest_step]
+
+ async def get_actions(task_id: str, organization_id: str | None = None) -> list[ClickAction]:
+ assert organization_id == "org-1"
+ return [older_action] if task_id == "task-a" else [latest_action]
+
+ tasks_repo = SimpleNamespace(
+ get_tasks_by_workflow_run_id=AsyncMock(return_value=[older_task, latest_task]),
+ get_task_steps=AsyncMock(side_effect=get_steps),
+ get_task_actions_hydrated=AsyncMock(side_effect=get_actions),
+ get_run=AsyncMock(return_value=SimpleNamespace(task_run_type=RunType.task_v1)),
+ )
+ monkeypatch.setattr(
+ shadow.app,
+ "DATABASE",
+ SimpleNamespace(
+ tasks=tasks_repo,
+ artifacts=SimpleNamespace(get_artifacts_for_task_step=AsyncMock(return_value=[])),
+ observer=SimpleNamespace(get_workflow_run_block_by_task_id=AsyncMock(return_value=None)),
+ workflow_runs=SimpleNamespace(get_workflow_run_output_parameters=AsyncMock(return_value=[])),
+ ),
+ )
+ monkeypatch.setattr(shadow.app, "ARTIFACT_MANAGER", SimpleNamespace(retrieve_artifact=AsyncMock()))
+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=_har(offset_seconds=12),
+ browser_state=_browser_state(),
+ last_step=latest_step,
+ workflow_run=_workflow_run(),
+ )
+
+ event = next(record for record in logs if record["event"] == "submission_signal_shadow")
+ assert event["signal"] == "submitted_verified"
+ assert event["task_id"] == "task-a"
+ tasks_repo.get_run.assert_awaited_once_with(run_id="task-b", organization_id="org-1")
+
+
+async def test_runner_swallows_errors_without_logging_exception_messages(monkeypatch: pytest.MonkeyPatch) -> None:
+ secret = "raw-value?token=do-not-log"
+ database = SimpleNamespace(
+ tasks=SimpleNamespace(get_task_steps=AsyncMock(side_effect=ValueError(secret))),
+ artifacts=SimpleNamespace(),
+ observer=SimpleNamespace(),
+ )
+ monkeypatch.setattr(shadow.app, "DATABASE", database)
+
+ with capture_logs() as logs:
+ await shadow.run_submission_signal_shadow(
+ har_data=b"",
+ browser_state=_browser_state(),
+ last_step=_step("step-1", 1),
+ task=_task(),
+ )
+
+ events = [record for record in logs if record["event"] == "submission_signal_shadow"]
+ assert len(events) == 1
+ assert events[0]["status"] == "error"
+ assert events[0]["error_stage"] == "load_tasks"
+ assert events[0]["error_type"] == "ValueError"
+ assert secret not in str(events[0])
+ assert "error" not in events[0] or events[0]["error"] != secret
diff --git a/tests/unit/submission/test_verifier.py b/tests/unit/submission/test_verifier.py
new file mode 100644
index 000000000..ce9d960bf
--- /dev/null
+++ b/tests/unit/submission/test_verifier.py
@@ -0,0 +1,779 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timedelta
+
+import pytest
+
+from skyvern.forge.sdk.models import Step, StepStatus
+from skyvern.forge.sdk.submission.models import (
+ BrowserPath,
+ CandidateWindow,
+ DownloadEvidence,
+ NetworkEvidence,
+ PageConfirmationEvidence,
+ SubmissionSignal,
+ TierAEvaluation,
+ TierBEvaluation,
+ UrlTransitionEvidence,
+)
+from skyvern.forge.sdk.submission.verifier import (
+ CandidateEvaluation,
+ build_candidate_windows,
+ classify_browser_path,
+ combine,
+ detect_submit_candidates,
+ evaluate_tier_a,
+ evaluate_tier_b,
+ find_latest_candidate_step_pair,
+)
+from skyvern.schemas.steps import AgentStepOutput, BrowserMetadata
+from skyvern.webeye.actions.actions import ClickAction
+from skyvern.webeye.actions.responses import ActionResult
+
+BASE_TIME = datetime(2026, 7, 9, 12, 0, 0)
+
+
+def _step(step_id: str, order: int, *, url: str | None = None) -> Step:
+ created_at = BASE_TIME + timedelta(seconds=order * 20)
+ return Step(
+ created_at=created_at,
+ modified_at=created_at + timedelta(seconds=5),
+ task_id="task_1",
+ step_id=step_id,
+ status=StepStatus.completed,
+ output=AgentStepOutput(browser_metadata=BrowserMetadata(website_url=url)),
+ order=order,
+ is_last=False,
+ organization_id="org_1",
+ )
+
+
+def _submit_click(*, step_id: str = "step_1", text: str = "Submit") -> ClickAction:
+ return ClickAction(
+ element_id="button_1",
+ reasoning="submit",
+ action_id="action_1",
+ task_id="task_1",
+ step_id=step_id,
+ skyvern_element_data={"tagName": "button", "attributes": {}, "text": text},
+ )
+
+
+def _tier_a(
+ *,
+ evidence: list[NetworkEvidence] | None = None,
+ har_present: bool = True,
+ har_parsed: bool = True,
+ har_entry_count: int = 1,
+ ambiguous_entry_count: int = 0,
+) -> TierAEvaluation:
+ return TierAEvaluation(
+ evidence=evidence or [],
+ har_present=har_present,
+ har_parsed=har_parsed,
+ har_entry_count=har_entry_count,
+ ambiguous_entry_count=ambiguous_entry_count,
+ )
+
+
+def _tier_b(
+ evidence: list[PageConfirmationEvidence | UrlTransitionEvidence | DownloadEvidence] | None = None,
+ *,
+ page_confirmation_evaluated: bool = True,
+) -> TierBEvaluation:
+ return TierBEvaluation(
+ evidence=evidence or [],
+ page_confirmation_evaluated=page_confirmation_evaluated,
+ )
+
+
+def _network_evidence() -> NetworkEvidence:
+ return NetworkEvidence(
+ origin="https://example.com",
+ method="POST",
+ status=201,
+ started_at=BASE_TIME,
+ correlated_step_id="step_1",
+ )
+
+
+def _page_evidence() -> PageConfirmationEvidence:
+ return PageConfirmationEvidence(
+ phrase="confirmation number",
+ value_sha256=hashlib.sha256(b"abc-123").hexdigest(),
+ absent_pre_submit=True,
+ )
+
+
+def _har_for_candidate(*, method: str = "POST", status: int = 201) -> bytes:
+ return json.dumps(
+ {
+ "log": {
+ "entries": [
+ {
+ "startedDateTime": "2026-07-09T12:00:22Z",
+ "request": {"method": method, "url": "https://example.com/api/submit"},
+ "response": {"status": status, "content": {"mimeType": "application/json"}},
+ "_resourceType": "xhr",
+ }
+ ]
+ }
+ }
+ ).encode()
+
+
+def _har_at(*offsets: int) -> bytes:
+ return json.dumps(
+ {
+ "log": {
+ "entries": [
+ {
+ "startedDateTime": (BASE_TIME + timedelta(seconds=offset)).isoformat() + "Z",
+ "request": {"method": "POST", "url": "https://example.com/api/submit"},
+ "response": {"status": 201, "content": {"mimeType": "application/json"}},
+ "_resourceType": "xhr",
+ }
+ for offset in offsets
+ ]
+ }
+ }
+ ).encode()
+
+
+def _window(step_id: str, started_at: int, ended_at: int) -> CandidateWindow:
+ return CandidateWindow(
+ step_id=step_id,
+ started_at=BASE_TIME + timedelta(seconds=started_at),
+ ended_at=BASE_TIME + timedelta(seconds=ended_at),
+ )
+
+
+def _combine(
+ *,
+ tier_a: TierAEvaluation | None = None,
+ tier_b: TierBEvaluation | None = None,
+ submit_intent_detected: bool = True,
+ browser_path: BrowserPath = BrowserPath.SKYVERN_CREATED,
+ cua_run: bool = False,
+ coordinate_click: bool = False,
+):
+ tier_a = tier_a or _tier_a()
+ return combine(
+ tier_a=tier_a,
+ candidate_evaluations=[
+ CandidateEvaluation(
+ step_id="step_1",
+ tier_a=tier_a.evidence,
+ tier_b=tier_b or _tier_b(),
+ is_latest=True,
+ )
+ ],
+ detected_candidate_step_ids=["step_1"],
+ submit_intent_detected=submit_intent_detected,
+ browser_path=browser_path,
+ cua_run=cua_run,
+ coordinate_click=coordinate_click,
+ )
+
+
+def test_classify_browser_path_uses_first_matching_signal() -> None:
+ assert (
+ classify_browser_path(
+ browser_session_id="session_1",
+ task_browser_session_id=None,
+ remote_browser_session_id="remote_1",
+ task_browser_address="ws://example.test",
+ needs_cdp_frame_publisher=True,
+ browser_type="chromium-headless",
+ )
+ == BrowserPath.SESSION_ATTACHED
+ )
+ assert (
+ classify_browser_path(
+ browser_session_id=None,
+ task_browser_session_id=None,
+ remote_browser_session_id="remote_1",
+ task_browser_address="ws://example.test",
+ needs_cdp_frame_publisher=True,
+ browser_type="chromium-headless",
+ )
+ == BrowserPath.VENDOR_REUSED
+ )
+ assert (
+ classify_browser_path(
+ browser_session_id=None,
+ task_browser_session_id=None,
+ remote_browser_session_id=None,
+ task_browser_address=None,
+ needs_cdp_frame_publisher=True,
+ browser_type="chromium-headless",
+ )
+ == BrowserPath.CDP_CONNECT
+ )
+ assert (
+ classify_browser_path(
+ browser_session_id=None,
+ task_browser_session_id=None,
+ remote_browser_session_id=None,
+ task_browser_address=None,
+ needs_cdp_frame_publisher=False,
+ browser_type="chromium-headful",
+ )
+ == BrowserPath.SKYVERN_CREATED
+ )
+ assert (
+ classify_browser_path(
+ browser_session_id=None,
+ task_browser_session_id=None,
+ remote_browser_session_id=None,
+ task_browser_address=None,
+ needs_cdp_frame_publisher=False,
+ browser_type="other",
+ )
+ == BrowserPath.UNKNOWN
+ )
+
+
+def test_detect_submit_candidates_and_coordinate_click_override() -> None:
+ typed_submit = ClickAction(
+ element_id="button_1",
+ reasoning="submit",
+ step_id="step_1",
+ skyvern_element_data={"tagName": "input", "attributes": {"type": "submit"}, "text": "Continue"},
+ )
+ verb_submit = _submit_click(text="Apply now")
+ false_boundary = _submit_click(text="Resubmit")
+ split_phrase = ClickAction(
+ element_id="button_2",
+ reasoning="continue",
+ step_id="step_1",
+ skyvern_element_data={
+ "tagName": "button",
+ "attributes": {"aria-label": "order"},
+ "text": "Place",
+ },
+ )
+ coordinate_click = ClickAction(
+ element_id="button_3",
+ reasoning="coordinate",
+ step_id="step_2",
+ x=10,
+ skyvern_element_data={"tagName": "button", "attributes": {"type": "submit"}, "text": "Submit"},
+ )
+
+ detected = detect_submit_candidates([typed_submit, verb_submit, false_boundary, split_phrase, coordinate_click])
+
+ assert [candidate.step_id for candidate in detected.candidates] == ["step_1", "step_1"]
+ assert detected.submit_intent_detected is True
+ assert detected.coordinate_click is True
+ assert "Apply now" not in detected.model_dump_json()
+
+
+def test_combine_uses_the_strongest_coherent_candidate() -> None:
+ older_network = _network_evidence()
+ latest_page = _page_evidence()
+ latest_download = DownloadEvidence(file_count=1)
+ tier_a = _tier_a(evidence=[older_network])
+
+ verdict = combine(
+ tier_a=tier_a,
+ candidate_evaluations=[
+ CandidateEvaluation(
+ step_id="step_1",
+ tier_a=[older_network],
+ tier_b=_tier_b(),
+ ),
+ CandidateEvaluation(
+ step_id="step_3",
+ tier_a=[],
+ tier_b=_tier_b([latest_page]),
+ is_latest=True,
+ ),
+ ],
+ detected_candidate_step_ids=["step_1", "step_3"],
+ submit_intent_detected=True,
+ browser_path=BrowserPath.SKYVERN_CREATED,
+ )
+
+ assert verdict.signal == SubmissionSignal.SUBMITTED_LIKELY
+ assert verdict.tier_a == []
+ assert verdict.tier_b == [latest_page]
+ assert verdict.winning_step_id == "step_3"
+ assert not any(note.startswith("winning_step_id:") for note in verdict.notes)
+
+ verdict = combine(
+ tier_a=tier_a,
+ candidate_evaluations=[
+ CandidateEvaluation(
+ step_id="step_1",
+ tier_a=[older_network],
+ tier_b=_tier_b([latest_page]),
+ ),
+ CandidateEvaluation(
+ step_id="step_3",
+ tier_a=[],
+ tier_b=_tier_b([latest_download]),
+ is_latest=True,
+ ),
+ ],
+ detected_candidate_step_ids=["step_1", "step_3"],
+ submit_intent_detected=True,
+ browser_path=BrowserPath.SKYVERN_CREATED,
+ )
+
+ assert verdict.signal == SubmissionSignal.SUBMITTED_VERIFIED
+ assert verdict.tier_a == [older_network]
+ assert verdict.tier_b == [latest_page]
+ assert verdict.winning_step_id == "step_1"
+ assert not any(note.startswith("winning_step_id:") for note in verdict.notes)
+
+
+def test_tier_a_discards_an_entry_matching_multiple_candidate_windows() -> None:
+ windows = [_window("step_1", 10, 25), _window("step_3", 20, 35)]
+ tier_a = evaluate_tier_a(_har_at(22), windows)
+
+ verdict = combine(
+ tier_a=tier_a,
+ candidate_evaluations=[
+ CandidateEvaluation(
+ step_id="step_1",
+ tier_a=[item for item in tier_a.evidence if item.correlated_step_id == "step_1"],
+ tier_b=_tier_b([_page_evidence()]),
+ ),
+ CandidateEvaluation(
+ step_id="step_3",
+ tier_a=[item for item in tier_a.evidence if item.correlated_step_id == "step_3"],
+ tier_b=_tier_b(),
+ is_latest=True,
+ ),
+ ],
+ detected_candidate_step_ids=["step_1", "step_3"],
+ submit_intent_detected=True,
+ browser_path=BrowserPath.SKYVERN_CREATED,
+ )
+
+ assert verdict.signal == SubmissionSignal.SUBMITTED_LIKELY
+ assert tier_a.evidence == []
+ assert verdict.tier_a == []
+ assert verdict.winning_step_id == "step_1"
+ assert "ambiguous_entries:1" in verdict.notes
+
+
+def test_tier_a_attributes_an_entry_matching_exactly_one_of_overlapping_windows() -> None:
+ windows = [_window("step_1", 10, 25), _window("step_3", 20, 35)]
+
+ evaluation = evaluate_tier_a(_har_at(12), windows)
+
+ assert [item.correlated_step_id for item in evaluation.evidence] == ["step_1"]
+ assert evaluation.ambiguous_entry_count == 0
+
+
+def test_tier_a_attributes_two_unambiguous_entries_to_different_candidates() -> None:
+ windows = [_window("step_1", 10, 25), _window("step_3", 20, 35)]
+
+ evaluation = evaluate_tier_a(_har_at(12, 32), windows)
+
+ assert [item.correlated_step_id for item in evaluation.evidence] == ["step_1", "step_3"]
+ assert evaluation.ambiguous_entry_count == 0
+
+
+def test_candidate_pair_uses_next_chronological_step_as_post_action() -> None:
+ steps = [_step("step_3", 3), _step("step_1", 1), _step("step_2", 2)]
+ detected = detect_submit_candidates([_submit_click(step_id="step_2")])
+
+ windows = build_candidate_windows(detected, steps)
+ pre_step, post_step = find_latest_candidate_step_pair(detected, steps)
+
+ assert len(windows) == 1
+ assert windows[0].step_id == "step_2"
+ assert windows[0].started_at < steps[2].created_at
+ assert pre_step is not None and pre_step.step_id == "step_2"
+ assert post_step is not None and post_step.step_id == "step_3"
+
+
+def test_latest_candidate_without_a_post_step_does_not_reuse_an_older_pair() -> None:
+ steps = [_step("step_1", 1), _step("step_2", 2), _step("step_3", 3)]
+ detected = detect_submit_candidates([_submit_click(step_id="step_1"), _submit_click(step_id="step_3")])
+
+ pre_step, post_step = find_latest_candidate_step_pair(detected, steps)
+
+ assert pre_step is not None and pre_step.step_id == "step_3"
+ assert post_step is None
+
+
+def test_tier_a_filters_non_submission_entries_and_normalizes_timestamps() -> None:
+ step = _step("step_1", 1)
+ windows = build_candidate_windows(detect_submit_candidates([_submit_click()]), [step])
+
+ def entry(
+ url: str,
+ *,
+ method: str = "POST",
+ status: int = 200,
+ resource_type: str | None = "xhr",
+ started_at: str = "2026-07-09T12:00:22Z",
+ mime_type: str = "application/json",
+ ) -> dict[str, object]:
+ value: dict[str, object] = {
+ "startedDateTime": started_at,
+ "request": {"method": method, "url": url},
+ "response": {"status": status, "content": {"mimeType": mime_type}},
+ }
+ if resource_type is not None:
+ value["_resourceType"] = resource_type
+ return value
+
+ har_bytes = json.dumps(
+ {
+ "log": {
+ "entries": [
+ entry("https://example.com/api/submit?token=secret", status=201),
+ entry("https://example.com/api/read", method="GET"),
+ entry("https://example.com/api/rejected", status=404),
+ entry("https://example.com/api/fail", status=500),
+ entry("https://example.com/image.png", resource_type="image", mime_type="image/png"),
+ entry("https://example.com/styles.css", resource_type=None, mime_type="text/css"),
+ entry(
+ "https://example.com/scripts/app.js",
+ resource_type=None,
+ mime_type="application/javascript",
+ ),
+ entry("https://example.com/api/late", started_at="2026-07-09T12:05:00+00:00"),
+ ]
+ }
+ }
+ ).encode()
+
+ evaluation = evaluate_tier_a(har_bytes, windows)
+
+ assert evaluation.har_present is True
+ assert evaluation.har_parsed is True
+ assert evaluation.har_entry_count == 8
+ assert len(evaluation.evidence) == 1
+ assert evaluation.evidence[0].origin == "https://example.com"
+ assert evaluation.evidence[0].correlated_step_id == "step_1"
+
+
[email protected](("method", "status"), [("POST", 201), ("PUT", 204), ("PATCH", 302)])
+def test_tier_a_accepts_each_submission_method_and_success_status(method: str, status: int) -> None:
+ step = _step("step_1", 1)
+ windows = build_candidate_windows(detect_submit_candidates([_submit_click()]), [step])
+ har_bytes = json.dumps(
+ {
+ "log": {
+ "entries": [
+ {
+ "startedDateTime": "2026-07-09T12:00:22Z",
+ "request": {"method": method, "url": "https://example.com/api/submit"},
+ "response": {"status": status, "content": {"mimeType": "application/json"}},
+ "_resourceType": "fetch",
+ }
+ ]
+ }
+ }
+ ).encode()
+
+ evaluation = evaluate_tier_a(har_bytes, windows)
+
+ assert [item.method for item in evaluation.evidence] == [method]
+
+
+def test_tier_a_malformed_har_is_unavailable_not_an_exception() -> None:
+ evaluation = evaluate_tier_a(b"not-json", [])
+
+ assert evaluation.har_present is True
+ assert evaluation.har_parsed is False
+ assert evaluation.har_entry_count == 0
+ assert evaluation.evidence == []
+
+
+def test_tier_a_excessively_nested_har_is_unavailable_not_an_exception() -> None:
+ deeply_nested_json = b'{"x":' * 10_000 + b"0" + b"}" * 10_000
+
+ evaluation = evaluate_tier_a(deeply_nested_json, [])
+
+ assert evaluation.har_present is True
+ assert evaluation.har_parsed is False
+ assert evaluation.har_entry_count == 0
+
+
+def test_tier_a_empty_har_is_missing() -> None:
+ evaluation = evaluate_tier_a(b"", [])
+
+ assert evaluation.har_present is False
+ assert evaluation.har_parsed is False
+ assert evaluation.har_entry_count == 0
+
+
+def test_tier_b_detects_independent_evidence_without_retaining_raw_values() -> None:
+ fake_value = "abc-123"
+ evaluation = evaluate_tier_b(
+ pre_url="https://example.test/apply/[email protected]/token-before?draft=private#form",
+ post_url="https://example.test/complete/[email protected]/token-after?token=private#receipt",
+ action_results=[ActionResult(success=True, download_triggered=True, downloaded_files=["private-receipt.pdf"])],
+ pre_page_text="Application ready",
+ post_page_text=f"Thank you. Confirmation number: {fake_value}",
+ )
+
+ assert evaluation.page_confirmation_evaluated is True
+ assert len(evaluation.evidence) == 3
+ url_evidence = next(evidence for evidence in evaluation.evidence if isinstance(evidence, UrlTransitionEvidence))
+ confirmation = next(evidence for evidence in evaluation.evidence if isinstance(evidence, PageConfirmationEvidence))
+ download = next(evidence for evidence in evaluation.evidence if isinstance(evidence, DownloadEvidence))
+ assert url_evidence.from_origin == "https://example.test"
+ assert url_evidence.to_origin == "https://example.test"
+ assert url_evidence.path_changed is True
+ assert confirmation.value_sha256 == hashlib.sha256(fake_value.encode()).hexdigest()
+ assert download.file_count == 1
+ assert fake_value not in evaluation.model_dump_json()
+ assert "private-receipt.pdf" not in evaluation.model_dump_json()
+ assert "token=" not in evaluation.model_dump_json()
+ assert "[email protected]" not in evaluation.model_dump_json()
+ assert "token-before" not in evaluation.model_dump_json()
+ assert "token-after" not in evaluation.model_dump_json()
+
+
+def test_tier_b_ignores_query_fragment_only_url_changes_and_existing_confirmation() -> None:
+ evaluation = evaluate_tier_b(
+ pre_url="https://example.test/form?draft=one#start",
+ post_url="https://example.test/form?draft=two#end",
+ action_results=[],
+ pre_page_text="Thank you for reviewing your request.",
+ post_page_text="Thank you for reviewing your request.",
+ )
+
+ assert evaluation.page_confirmation_evaluated is True
+ assert evaluation.evidence == []
+
+
+ "result",
+ [
+ ActionResult(success=True, download_triggered=True),
+ ActionResult(success=True, downloaded_files=["receipt.pdf"]),
+ ],
+)
+def test_tier_b_accepts_each_download_signal(result: ActionResult) -> None:
+ evaluation = evaluate_tier_b(
+ pre_url=None,
+ post_url=None,
+ action_results=[result],
+ pre_page_text="Ready",
+ post_page_text="Still ready",
+ )
+
+ assert evaluation.evidence == [DownloadEvidence(file_count=1)]
+
+
+def test_all_submission_signal_values_are_reachable() -> None:
+ assert (
+ _combine(tier_a=_tier_a(evidence=[_network_evidence()]), tier_b=_tier_b([_page_evidence()])).signal
+ == SubmissionSignal.SUBMITTED_VERIFIED
+ )
+ assert _combine(tier_a=_tier_a(evidence=[_network_evidence()])).signal == SubmissionSignal.SUBMITTED_LIKELY
+ assert _combine(tier_b=_tier_b([_page_evidence()])).signal == SubmissionSignal.SUBMITTED_LIKELY
+ assert _combine().signal == SubmissionSignal.NOT_SUBMITTED
+ assert _combine(tier_a=_tier_a(har_present=False, har_parsed=False, har_entry_count=0)).signal == (
+ SubmissionSignal.NOT_EVALUATED
+ )
+
+
+ ("har_bytes", "post_page_text", "expected_signal"),
+ [
+ (_har_for_candidate(), "Thank you. Your request was received.", SubmissionSignal.SUBMITTED_VERIFIED),
+ (_har_for_candidate(), "The form remains ready.", SubmissionSignal.SUBMITTED_LIKELY),
+ (_har_for_candidate(method="GET"), "The form remains ready.", SubmissionSignal.NOT_SUBMITTED),
+ (b"", "The form remains ready.", SubmissionSignal.NOT_EVALUATED),
+ ],
+)
+def test_every_signal_is_reachable_from_raw_inputs(
+ har_bytes: bytes,
+ post_page_text: str,
+ expected_signal: SubmissionSignal,
+) -> None:
+ step = _step("step_1", 1)
+ detection = detect_submit_candidates([_submit_click()])
+ tier_a = evaluate_tier_a(har_bytes, build_candidate_windows(detection, [step]))
+ tier_b = evaluate_tier_b(
+ pre_url="https://example.com/form",
+ post_url="https://example.com/form",
+ action_results=[],
+ pre_page_text="Review your request.",
+ post_page_text=post_page_text,
+ )
+
+ verdict = combine(
+ tier_a=tier_a,
+ candidate_evaluations=[
+ CandidateEvaluation(
+ step_id="step_1",
+ tier_a=tier_a.evidence,
+ tier_b=tier_b,
+ is_latest=True,
+ )
+ ],
+ detected_candidate_step_ids=["step_1"],
+ submit_intent_detected=detection.submit_intent_detected,
+ browser_path=BrowserPath.SKYVERN_CREATED,
+ )
+
+ assert verdict.signal == expected_signal
+
+
+def test_not_submitted_unreachable_with_ambiguous_entries() -> None:
+ verdict = _combine(tier_a=_tier_a(ambiguous_entry_count=1))
+
+ assert verdict.signal == SubmissionSignal.NOT_EVALUATED
+ assert "ambiguous_entries:1" in verdict.notes
+
+
+def test_not_submitted_unreachable_when_any_candidate_page_confirmation_unknown() -> None:
+ older = CandidateEvaluation(step_id="step_1", tier_a=[], tier_b=_tier_b(page_confirmation_evaluated=False))
+ latest = CandidateEvaluation(step_id="step_2", tier_a=[], tier_b=_tier_b(), is_latest=True)
+
+ verdict = combine(
+ tier_a=_tier_a(),
+ candidate_evaluations=[older, latest],
+ detected_candidate_step_ids=["step_1", "step_2"],
+ submit_intent_detected=True,
+ browser_path=BrowserPath.SKYVERN_CREATED,
+ )
+
+ assert verdict.signal == SubmissionSignal.NOT_EVALUATED
+
+
+def test_not_submitted_unreachable_when_a_detected_candidate_step_is_unmapped() -> None:
+ mapped = CandidateEvaluation(step_id="step_valid", tier_a=[], tier_b=_tier_b(), is_latest=True)
+
+ verdict = combine(
+ tier_a=_tier_a(),
+ candidate_evaluations=[mapped],
+ detected_candidate_step_ids=["step_valid", "step_missing"],
+ submit_intent_detected=True,
+ browser_path=BrowserPath.SKYVERN_CREATED,
+ )
+
+ assert verdict.signal == SubmissionSignal.NOT_EVALUATED
+
+
+def test_not_submitted_unreachable_when_har_is_missing() -> None:
+ assert _combine(tier_a=_tier_a(har_present=False)).signal == SubmissionSignal.NOT_EVALUATED
+
+
+def test_not_submitted_unreachable_when_har_is_unparseable() -> None:
+ assert _combine(tier_a=_tier_a(har_parsed=False)).signal == SubmissionSignal.NOT_EVALUATED
+
+
+def test_not_submitted_unreachable_when_har_has_zero_entries() -> None:
+ assert _combine(tier_a=_tier_a(har_entry_count=0)).signal == SubmissionSignal.NOT_EVALUATED
+
+
+def test_not_submitted_unreachable_without_submit_intent() -> None:
+ assert _combine(submit_intent_detected=False).signal == SubmissionSignal.NOT_EVALUATED
+
+
+def test_not_submitted_unreachable_outside_skyvern_created_path() -> None:
+ assert _combine(browser_path=BrowserPath.CDP_CONNECT).signal == SubmissionSignal.NOT_EVALUATED
+
+
+def test_not_submitted_unreachable_when_page_confirmation_is_unavailable() -> None:
+ assert _combine(tier_b=_tier_b(page_confirmation_evaluated=False)).signal == SubmissionSignal.NOT_EVALUATED
+
+
[email protected]("override", [{"cua_run": True}, {"coordinate_click": True}])
+def test_cua_and_coordinate_clicks_are_not_evaluated(override: dict[str, bool]) -> None:
+ verdict = _combine(
+ tier_a=_tier_a(evidence=[_network_evidence()]),
+ tier_b=_tier_b([_page_evidence()]),
+ **override,
+ )
+
+ assert verdict.signal == SubmissionSignal.NOT_EVALUATED
+
+
+ "browser_path",
+ [BrowserPath.CDP_CONNECT, BrowserPath.VENDOR_REUSED, BrowserPath.SESSION_ATTACHED, BrowserPath.UNKNOWN],
+)
+def test_non_skyvern_paths_are_tier_b_only_and_capped(browser_path: BrowserPath) -> None:
+ verdict = _combine(
+ tier_a=_tier_a(evidence=[_network_evidence()]),
+ tier_b=_tier_b([_page_evidence(), DownloadEvidence(file_count=1)]),
+ browser_path=browser_path,
+ )
+
+ assert verdict.signal == SubmissionSignal.SUBMITTED_LIKELY
+ assert verdict.tier_a == []
+ assert verdict.capped is True
+
+ without_tier_b = _combine(
+ tier_a=_tier_a(evidence=[_network_evidence()]),
+ browser_path=browser_path,
+ )
+ assert without_tier_b.signal == SubmissionSignal.NOT_EVALUATED
+ assert without_tier_b.tier_a == []
+ assert without_tier_b.capped is True
+
+
+def test_evidence_models_only_serialize_derived_facts() -> None:
+ fake_value = "fake-confirmation-value"
+ fake_filename = "private-receipt.pdf"
+ har_bytes = json.dumps(
+ {
+ "log": {
+ "entries": [
+ {
+ "startedDateTime": "2026-07-09T12:00:22Z",
+ "request": {
+ "method": "POST",
+ "url": "https://example.com/api/submit?token=private",
+ "postData": {"text": fake_value},
+ },
+ "response": {
+ "status": 201,
+ "content": {"mimeType": "application/json", "text": fake_value},
+ },
+ "_resourceType": "fetch",
+ }
+ ]
+ }
+ }
+ ).encode()
+ step = _step("step_1", 1)
+ detection = detect_submit_candidates([_submit_click()])
+ tier_a = evaluate_tier_a(har_bytes, build_candidate_windows(detection, [step]))
+ tier_b = evaluate_tier_b(
+ pre_url="https://example.com/form/[email protected]/private-token?draft=private#start",
+ post_url="https://example.com/complete/[email protected]/private-token?token=private#receipt",
+ action_results=[ActionResult(success=True, downloaded_files=[fake_filename])],
+ pre_page_text="Review your request.",
+ post_page_text=f"Confirmation number: {fake_value}",
+ )
+ verdict = combine(
+ tier_a=tier_a,
+ candidate_evaluations=[
+ CandidateEvaluation(
+ step_id="step_1",
+ tier_a=tier_a.evidence,
+ tier_b=tier_b,
+ is_latest=True,
+ )
+ ],
+ detected_candidate_step_ids=["step_1"],
+ submit_intent_detected=detection.submit_intent_detected,
+ browser_path=BrowserPath.SKYVERN_CREATED,
+ )
+
+ serialized = verdict.model_dump_json()
+ assert verdict.signal == SubmissionSignal.SUBMITTED_VERIFIED
+ assert fake_value not in serialized
+ assert fake_filename not in serialized
+ assert "?" not in serialized
+ assert "body" not in serialized
+ assert "postData" not in serialized
+ assert "[email protected]" not in serialized
+ assert "private-token" not in serialized
+ assert hashlib.sha256(fake_value.encode()).hexdigest() in serialized
diff --git a/tests/unit/submission/test_wiring.py b/tests/unit/submission/test_wiring.py
new file mode 100644
index 000000000..35e97b69d
--- /dev/null
+++ b/tests/unit/submission/test_wiring.py
@@ -0,0 +1,313 @@
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.config import Settings
+from skyvern.forge.agent import ForgeAgent
+from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.webeye.browser_artifacts import BrowserArtifacts
+
+
+def _task(task_id: str = "tsk_1", organization_id: str = "o_1") -> MagicMock:
+ task = MagicMock()
+ task.task_id = task_id
+ task.organization_id = organization_id
+ task.webhook_callback_url = None
+ return task
+
+
+def _step(step_id: str = "stp_1", task_id: str = "tsk_1") -> MagicMock:
+ step = MagicMock()
+ step.step_id = step_id
+ step.task_id = task_id
+ return step
+
+
+def _browser_state() -> MagicMock:
+ browser_state = MagicMock()
+ browser_state.browser_artifacts = BrowserArtifacts()
+ browser_state.browser_context = None
+ return browser_state
+
+
+def _workflow() -> MagicMock:
+ workflow = MagicMock()
+ workflow.workflow_id = "wf_1"
+ return workflow
+
+
+def _workflow_run() -> MagicMock:
+ workflow_run = MagicMock()
+ workflow_run.workflow_run_id = "wr_1"
+ return workflow_run
+
+
+def test_submission_signal_shadow_setting_defaults_off() -> None:
+ assert Settings.model_fields["SKYVERN_SUBMISSION_SIGNAL_SHADOW"].default is False
+
+
+async def test_task_cleanup_does_not_schedule_submission_shadow_when_flag_is_off() -> None:
+ agent = ForgeAgent()
+ task = _task()
+ last_step = _step()
+ browser_state = _browser_state()
+ scheduled: list[dict[str, object]] = []
+
+ def schedule_submission_signal_shadow(**kwargs: object) -> None:
+ scheduled.append(kwargs)
+
+ with (
+ patch("skyvern.forge.agent.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", False),
+ patch(
+ "skyvern.forge.agent.submission_shadow.schedule_submission_signal_shadow",
+ side_effect=schedule_submission_signal_shadow,
+ ),
+ patch("skyvern.forge.agent.skyvern_context.current", return_value=None),
+ patch("skyvern.forge.agent.app") as mock_app,
+ ):
+ mock_app.BROWSER_MANAGER.cleanup_for_task = AsyncMock(return_value=browser_state)
+ mock_app.BROWSER_MANAGER.get_video_artifacts = AsyncMock(return_value=[])
+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(return_value=b'{"log":{"entries":[]}}')
+ mock_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")
+ mock_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()
+
+ await agent.cleanup_browser_and_create_artifacts(
+ close_browser_on_completion=True,
+ last_step=last_step,
+ task=task,
+ browser_session_id="pbs_1",
+ )
+
+ assert scheduled == []
+
+
+async def test_task_cleanup_schedules_submission_shadow_with_the_har_context() -> None:
+ agent = ForgeAgent()
+ task = _task()
+ last_step = _step()
+ browser_state = _browser_state()
+ har_data = b'{"log":{"entries":[]}}'
+ calls: list[str] = []
+ scheduled: list[dict[str, object]] = []
+
+ async def get_har_data(**_: object) -> bytes:
+ calls.append("har")
+ return har_data
+
+ async def get_browser_console_log(**_: object) -> bytes:
+ calls.append("browser_log")
+ return b""
+
+ def schedule_submission_signal_shadow(**kwargs: object) -> None:
+ calls.append("shadow")
+ scheduled.append(kwargs)
+
+ with (
+ patch("skyvern.forge.agent.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", True),
+ patch(
+ "skyvern.forge.agent.submission_shadow.schedule_submission_signal_shadow",
+ side_effect=schedule_submission_signal_shadow,
+ ),
+ patch("skyvern.forge.agent.skyvern_context.current", return_value=None),
+ patch("skyvern.forge.agent.app") as mock_app,
+ ):
+ mock_app.BROWSER_MANAGER.cleanup_for_task = AsyncMock(return_value=browser_state)
+ mock_app.BROWSER_MANAGER.get_video_artifacts = AsyncMock(return_value=[])
+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(side_effect=get_har_data)
+ mock_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(side_effect=get_browser_console_log)
+ mock_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()
+
+ await agent.cleanup_browser_and_create_artifacts(
+ close_browser_on_completion=True,
+ last_step=last_step,
+ task=task,
+ browser_session_id="pbs_1",
+ )
+
+ assert calls[:3] == ["har", "shadow", "browser_log"]
+ assert scheduled == [
+ {
+ "har_data": har_data,
+ "browser_state": browser_state,
+ "last_step": last_step,
+ "task": task,
+ "browser_session_id": "pbs_1",
+ }
+ ]
+
+
+async def test_workflow_har_cleanup_does_not_schedule_submission_shadow_when_flag_is_off() -> None:
+ service = WorkflowService()
+ browser_state = _browser_state()
+ last_step = _step()
+ workflow = _workflow()
+ workflow_run = _workflow_run()
+ scheduled: list[dict[str, object]] = []
+
+ def schedule_submission_signal_shadow(**kwargs: object) -> None:
+ scheduled.append(kwargs)
+
+ with (
+ patch("skyvern.forge.sdk.workflow.service.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", False),
+ patch(
+ "skyvern.forge.sdk.workflow.service.submission_shadow.schedule_submission_signal_shadow",
+ side_effect=schedule_submission_signal_shadow,
+ ),
+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,
+ ):
+ mock_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")
+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(return_value=b'{"log":{"entries":[]}}')
+ mock_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()
+ mock_app.ARTIFACT_MANAGER.create_task_archive = AsyncMock()
+
+ await service.persist_har_data(browser_state, last_step, workflow, workflow_run)
+ await service._persist_debug_artifacts_bundled(browser_state, last_step, workflow, workflow_run)
+
+ assert scheduled == []
+
+
+async def test_workflow_standalone_har_schedules_submission_shadow_with_the_har_context() -> None:
+ service = WorkflowService()
+ browser_state = _browser_state()
+ last_step = _step()
+ workflow = _workflow()
+ workflow_run = _workflow_run()
+ har_data = b'{"log":{"entries":[]}}'
+ calls: list[str] = []
+ scheduled: list[dict[str, object]] = []
+
+ async def get_har_data(**_: object) -> bytes:
+ calls.append("har")
+ return har_data
+
+ def schedule_submission_signal_shadow(**kwargs: object) -> None:
+ calls.append("shadow")
+ scheduled.append(kwargs)
+
+ async def create_artifact(**_: object) -> None:
+ calls.append("artifact")
+
+ with (
+ patch("skyvern.forge.sdk.workflow.service.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", True),
+ patch(
+ "skyvern.forge.sdk.workflow.service.submission_shadow.schedule_submission_signal_shadow",
+ side_effect=schedule_submission_signal_shadow,
+ ),
+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,
+ ):
+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(side_effect=get_har_data)
+ mock_app.ARTIFACT_MANAGER.create_artifact = AsyncMock(side_effect=create_artifact)
+
+ await service.persist_har_data(browser_state, last_step, workflow, workflow_run)
+
+ assert calls == ["har", "shadow", "artifact"]
+ assert scheduled == [
+ {
+ "har_data": har_data,
+ "browser_state": browser_state,
+ "last_step": last_step,
+ "workflow_run": workflow_run,
+ }
+ ]
+
+
+async def test_workflow_bundled_har_schedules_submission_shadow_with_the_har_context() -> None:
+ service = WorkflowService()
+ browser_state = _browser_state()
+ last_step = _step()
+ workflow = _workflow()
+ workflow_run = _workflow_run()
+ har_data = b'{"log":{"entries":[]}}'
+ calls: list[str] = []
+ scheduled: list[dict[str, object]] = []
+
+ async def get_har_data(**_: object) -> bytes:
+ calls.append("har")
+ return har_data
+
+ def schedule_submission_signal_shadow(**kwargs: object) -> None:
+ calls.append("shadow")
+ scheduled.append(kwargs)
+
+ async def create_task_archive(**_: object) -> None:
+ calls.append("archive")
+
+ with (
+ patch("skyvern.forge.sdk.workflow.service.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", True),
+ patch(
+ "skyvern.forge.sdk.workflow.service.submission_shadow.schedule_submission_signal_shadow",
+ side_effect=schedule_submission_signal_shadow,
+ ),
+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,
+ ):
+ mock_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")
+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(side_effect=get_har_data)
+ mock_app.ARTIFACT_MANAGER.create_task_archive = AsyncMock(side_effect=create_task_archive)
+
+ await service._persist_debug_artifacts_bundled(browser_state, last_step, workflow, workflow_run)
+
+ assert calls == ["har", "shadow", "archive"]
+ assert scheduled == [
+ {
+ "har_data": har_data,
+ "browser_state": browser_state,
+ "last_step": last_step,
+ "workflow_run": workflow_run,
+ }
+ ]
+
+
+async def test_wiring_persists_har_when_shadow_scheduler_setup_fails() -> None:
+ agent = ForgeAgent()
+ service = WorkflowService()
+ task = _task()
+ browser_state = _browser_state()
+ last_step = _step()
+ workflow = _workflow()
+ workflow_run = _workflow_run()
+ har_data = b'{"log":{"entries":[]}}'
+
+ with (
+ patch("skyvern.forge.sdk.submission.shadow.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", True),
+ patch(
+ "skyvern.forge.sdk.submission.shadow._prune_pending",
+ side_effect=RuntimeError("scheduler setup failed"),
+ ),
+ patch("skyvern.forge.agent.skyvern_context.current", return_value=None),
+ patch("skyvern.forge.agent.app") as agent_app,
+ patch("skyvern.forge.sdk.workflow.service.app") as workflow_app,
+ ):
+ agent_app.BROWSER_MANAGER.cleanup_for_task = AsyncMock(return_value=browser_state)
+ agent_app.BROWSER_MANAGER.get_video_artifacts = AsyncMock(return_value=[])
+ agent_app.BROWSER_MANAGER.get_har_data = AsyncMock(return_value=har_data)
+ agent_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")
+ agent_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()
+ workflow_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")
+ workflow_app.BROWSER_MANAGER.get_har_data = AsyncMock(return_value=har_data)
+ workflow_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()
+ workflow_app.ARTIFACT_MANAGER.create_task_archive = AsyncMock()
+
+ await agent.cleanup_browser_and_create_artifacts(True, last_step, task, browser_session_id="pbs_1")
+ await service.persist_har_data(browser_state, last_step, workflow, workflow_run)
+ await service._persist_debug_artifacts_bundled(browser_state, last_step, workflow, workflow_run)
+
+ agent_app.ARTIFACT_MANAGER.create_artifact.assert_awaited_once_with(
+ step=last_step, artifact_type=ArtifactType.HAR, data=har_data
+ )
+ workflow_app.ARTIFACT_MANAGER.create_artifact.assert_awaited_once_with(
+ step=last_step, artifact_type=ArtifactType.HAR, data=har_data
+ )
+ workflow_app.ARTIFACT_MANAGER.create_task_archive.assert_awaited_once_with(
+ step=last_step,
+ entries={"har.har": (ArtifactType.HAR, har_data)},
+ workflow_run_id=workflow_run.workflow_run_id,
+ )