Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(submission): shadow-mode deterministic submission signal behind default-off flag (SKY-12162) (#7281)
Marc Kelechava31e6bdacda3e
diff --git a/skyvern/config.py b/skyvern/config.pyindex 171ef52da..a2d4900e8 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -145,6 +145,9 @@ class Settings(BaseSettings):# Static kill-switch for fail-fast shadow observability. Per-org rollout is the# PostHog flag FAIL_FAST_SHADOW; this only force-enables it everywhere (local/testing).FAIL_FAST_SHADOW: bool = False+ # Default-off telemetry for deterministic submission signals; enabling it only schedules+ # fire-and-forget shadow evaluation and does not change run behavior.+ SKYVERN_SUBMISSION_SIGNAL_SHADOW: bool = False# Global kill-switch for select/autocomplete shadow-match observability (LLM-vs-deterministic# agreement logging). Not per-org; set false to silence the logs everywhere.SKYVERN_SELECT_SHADOW_MATCH: bool = Truediff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex ef02e1fe9..358cafa68 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -108,6 +108,7 @@ from skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.tasks import Task, TaskRequest, TaskResponse, TaskStatusfrom skyvern.forge.sdk.schemas.totp_codes import OTPType+from skyvern.forge.sdk.submission import shadow as submission_shadowfrom skyvern.forge.sdk.trace import VerificationTrigger, apply_context_attrs, tracedfrom skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import (@@ -5006,6 +5007,14 @@ class ForgeAgent:_use_bundling = _ctx.use_artifact_bundling if _ctx else Falsehar_data = await app.BROWSER_MANAGER.get_har_data(task_id=task.task_id, browser_state=browser_state)+ if settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW:+ submission_shadow.schedule_submission_signal_shadow(+ har_data=har_data,+ browser_state=browser_state,+ last_step=last_step,+ task=task,+ browser_session_id=browser_session_id,+ )LOG.debug("Uploading har data", har_size=len(har_data))browser_log = await app.BROWSER_MANAGER.get_browser_console_log(diff --git a/skyvern/forge/sdk/submission/__init__.py b/skyvern/forge/sdk/submission/__init__.pynew file mode 100644index 000000000..a8c90cb29--- /dev/null+++ b/skyvern/forge/sdk/submission/__init__.py@@ -0,0 +1,47 @@+from skyvern.forge.sdk.submission.models import (+ BrowserPath,+ CandidateWindow,+ DownloadEvidence,+ NetworkEvidence,+ PageConfirmationEvidence,+ SubmissionSignal,+ SubmissionVerdict,+ SubmitCandidate,+ SubmitCandidateDetection,+ TierAEvaluation,+ TierBEvaluation,+ TierBEvidence,+ UrlTransitionEvidence,+)+from skyvern.forge.sdk.submission.verifier import (+ build_candidate_windows,+ classify_browser_path,+ combine,+ detect_submit_candidates,+ evaluate_tier_a,+ evaluate_tier_b,+ find_latest_candidate_step_pair,+)++__all__ = [+ "BrowserPath",+ "CandidateWindow",+ "DownloadEvidence",+ "NetworkEvidence",+ "PageConfirmationEvidence",+ "SubmissionSignal",+ "SubmissionVerdict",+ "SubmitCandidate",+ "SubmitCandidateDetection",+ "TierAEvaluation",+ "TierBEvaluation",+ "TierBEvidence",+ "UrlTransitionEvidence",+ "build_candidate_windows",+ "classify_browser_path",+ "combine",+ "detect_submit_candidates",+ "evaluate_tier_a",+ "evaluate_tier_b",+ "find_latest_candidate_step_pair",+]diff --git a/skyvern/forge/sdk/submission/models.py b/skyvern/forge/sdk/submission/models.pynew file mode 100644index 000000000..74117f103--- /dev/null+++ b/skyvern/forge/sdk/submission/models.py@@ -0,0 +1,111 @@+from __future__ import annotations++from datetime import datetime+from enum import StrEnum+from typing import TypeAlias++from pydantic import BaseModel, ConfigDict, Field+++class SubmissionSignal(StrEnum):+ SUBMITTED_VERIFIED = "submitted_verified"+ SUBMITTED_LIKELY = "submitted_likely"+ NOT_SUBMITTED = "not_submitted"+ NOT_EVALUATED = "not_evaluated"+++class BrowserPath(StrEnum):+ SKYVERN_CREATED = "skyvern_created"+ CDP_CONNECT = "cdp_connect"+ VENDOR_REUSED = "vendor_reused"+ SESSION_ATTACHED = "session_attached"+ UNKNOWN = "unknown"+++class SubmitCandidate(BaseModel):+ model_config = ConfigDict(frozen=True)++ action_id: str | None = None+ task_id: str | None = None+ step_id: str+++class SubmitCandidateDetection(BaseModel):+ candidates: list[SubmitCandidate]+ coordinate_click: bool = False++ @property+ def submit_intent_detected(self) -> bool:+ return bool(self.candidates)+++class CandidateWindow(BaseModel):+ model_config = ConfigDict(frozen=True)++ task_id: str | None = None+ step_id: str+ candidate_action_id: str | None = None+ started_at: datetime+ ended_at: datetime+++class NetworkEvidence(BaseModel):+ model_config = ConfigDict(frozen=True)++ origin: str+ method: str+ status: int+ started_at: datetime+ correlated_step_id: str+++class PageConfirmationEvidence(BaseModel):+ model_config = ConfigDict(frozen=True)++ phrase: str+ value_sha256: str+ absent_pre_submit: bool+++class UrlTransitionEvidence(BaseModel):+ model_config = ConfigDict(frozen=True)++ from_origin: str+ to_origin: str+ path_changed: bool+++class DownloadEvidence(BaseModel):+ model_config = ConfigDict(frozen=True)++ file_count: int+++TierBEvidence: TypeAlias = PageConfirmationEvidence | UrlTransitionEvidence | DownloadEvidence+++class TierAEvaluation(BaseModel):+ evidence: list[NetworkEvidence]+ har_present: bool+ har_parsed: bool+ har_entry_count: int+ ambiguous_entry_count: int = 0+++class TierBEvaluation(BaseModel):+ evidence: list[TierBEvidence]+ page_confirmation_evaluated: bool+++class SubmissionVerdict(BaseModel):+ signal: SubmissionSignal+ tier_a: list[NetworkEvidence]+ tier_b: list[TierBEvidence]+ submit_intent_detected: bool+ har_present: bool+ har_parsed: bool+ har_entry_count: int+ browser_path: BrowserPath+ winning_step_id: str | None = None+ capped: bool = False+ notes: list[str] = Field(default_factory=list)diff --git a/skyvern/forge/sdk/submission/shadow.py b/skyvern/forge/sdk/submission/shadow.pynew file mode 100644index 000000000..c6eec1563--- /dev/null+++ b/skyvern/forge/sdk/submission/shadow.py@@ -0,0 +1,390 @@+from __future__ import annotations++import asyncio+import time+from dataclasses import dataclass++import structlog++from skyvern.config import settings+from skyvern.forge import app+from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType+from skyvern.forge.sdk.models import Step+from skyvern.forge.sdk.schemas.tasks import Task+from skyvern.forge.sdk.submission.models import BrowserPath, CandidateWindow, SubmissionVerdict, TierAEvaluation+from skyvern.forge.sdk.submission.verifier import (+ CandidateEvaluation,+ build_candidate_windows,+ classify_browser_path,+ combine,+ detect_submit_candidates,+ evaluate_tier_a,+ evaluate_tier_b,+ find_candidate_step_pairs,+)+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun+from skyvern.services import service_utils+from skyvern.webeye.actions.actions import Action+from skyvern.webeye.actions.responses import ActionResult+from skyvern.webeye.browser_state import BrowserState++LOG = structlog.get_logger()++_SHADOW_EVENT = "submission_signal_shadow"+_ZIP_HEADER = b"PK\x03\x04"+_PAGE_ARTIFACT_TYPES = (+ ArtifactType.VISIBLE_ELEMENTS_TREE_IN_PROMPT,+ ArtifactType.HTML_SCRAPE,+)++_PENDING_SUBMISSION_SHADOW_TASKS: set[asyncio.Task[None]] = set()+_MAX_PENDING = 20+_MAX_HAR_BYTES = 25_000_000+++@dataclass(frozen=True)+class _LoadedContext:+ tasks: list[Task]+ steps: list[Step]+ actions: list[Action]+ organization_id: str+ workflow_run_id: str | None+ run_status: str+ output_is_none: bool+++def _elapsed_ms(started_at: float) -> int:+ return int((time.monotonic() - started_at) * 1000)+++def _prune_pending() -> None:+ _PENDING_SUBMISSION_SHADOW_TASKS.difference_update(+ {task for task in _PENDING_SUBMISSION_SHADOW_TASKS if task.done()}+ )+++def _track(task: asyncio.Task[None]) -> asyncio.Task[None]:+ _PENDING_SUBMISSION_SHADOW_TASKS.add(task)+ task.add_done_callback(_PENDING_SUBMISSION_SHADOW_TASKS.discard)+ return task+++def _merge_last_step(steps: list[Step], last_step: Step) -> list[Step]:+ merged = [last_step if step.step_id == last_step.step_id else step for step in steps]+ if all(step.step_id != last_step.step_id for step in steps):+ merged.append(last_step)+ return sorted(merged, key=lambda step: (step.created_at, step.order, step.retry_index))+++def _flatten_action_results(steps: list[Step]) -> list[ActionResult]:+ results: list[ActionResult] = []+ for step in steps:+ if step.output is None:+ continue+ if step.output.actions_and_results:+ results.extend(result for _, action_results in step.output.actions_and_results for result in action_results)+ elif step.output.action_results:+ results.extend(step.output.action_results)+ return results+++async def _load_context(+ *,+ task: Task | None,+ workflow_run: WorkflowRun | None,+ last_step: Step,+) -> _LoadedContext:+ if task is None and workflow_run is None:+ raise ValueError("task or workflow_run is required")+ if task is not None and workflow_run is not None:+ raise ValueError("task and workflow_run are mutually exclusive")++ if task is not None:+ tasks = [task]+ organization_id = task.organization_id+ workflow_run_id = task.workflow_run_id+ run_status = str(task.status)+ output_is_none = task.extracted_information is None+ else:+ assert workflow_run is not None+ tasks = await app.DATABASE.tasks.get_tasks_by_workflow_run_id(workflow_run_id=workflow_run.workflow_run_id)+ organization_id = workflow_run.organization_id+ workflow_run_id = workflow_run.workflow_run_id+ run_status = str(workflow_run.status)+ workflow_outputs = await app.DATABASE.workflow_runs.get_workflow_run_output_parameters(+ workflow_run_id=workflow_run.workflow_run_id+ )+ output_is_none = not workflow_outputs++ steps: list[Step] = []+ actions: list[Action] = []+ for item in tasks:+ task_steps = await app.DATABASE.tasks.get_task_steps(item.task_id, item.organization_id)+ steps.extend(task_steps)+ actions.extend(+ await app.DATABASE.tasks.get_task_actions_hydrated(+ item.task_id,+ organization_id=item.organization_id,+ )+ )++ if any(item.task_id == last_step.task_id for item in tasks):+ steps = _merge_last_step(steps, last_step)+ else:+ steps.sort(key=lambda step: (step.created_at, step.order, step.retry_index))++ return _LoadedContext(+ tasks=tasks,+ steps=steps,+ actions=actions,+ organization_id=organization_id,+ workflow_run_id=workflow_run_id,+ run_status=run_status,+ output_is_none=output_is_none,+ )+++def _preferred_page_artifact(artifacts: list[Artifact]) -> Artifact | None:+ for artifact_type in _PAGE_ARTIFACT_TYPES:+ matching = [artifact for artifact in artifacts if artifact.artifact_type == artifact_type]+ if matching:+ return max(matching, key=lambda artifact: artifact.created_at)+ return None+++async def _retrieve_page_text(step: Step) -> str | None:+ try:+ artifacts = await app.DATABASE.artifacts.get_artifacts_for_task_step(+ task_id=step.task_id,+ step_id=step.step_id,+ organization_id=step.organization_id,+ )+ artifact = _preferred_page_artifact(artifacts)+ if artifact is None:+ return None+ content = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)+ if not content or content.startswith(_ZIP_HEADER):+ return None+ return content.decode("utf-8")+ except Exception: # noqa: BLE001 - page confirmation is optional shadow evidence+ return None+++def _step_url(step: Step | None) -> str | None:+ if step is None or step.output is None or step.output.browser_metadata is None:+ return None+ return step.output.browser_metadata.website_url+++def _evaluate_submission_signal(+ *,+ har_data: bytes,+ candidate_windows: list[CandidateWindow],+ candidate_step_pairs: list[tuple[Step, Step | None]],+ detected_candidate_step_ids: list[str],+ latest_candidate_step: Step | None,+ pre_page_text: str | None,+ post_page_text: str | None,+ submit_intent_detected: bool,+ browser_path: BrowserPath,+ cua_run: bool,+ coordinate_click: bool,+) -> SubmissionVerdict:+ har_too_large = len(har_data) > _MAX_HAR_BYTES+ tier_a = (+ TierAEvaluation(evidence=[], har_present=True, har_parsed=False, har_entry_count=0)+ if har_too_large+ else evaluate_tier_a(har_data, candidate_windows)+ )+ candidate_evaluations = [+ CandidateEvaluation(+ step_id=candidate_step.step_id,+ tier_a=[evidence for evidence in tier_a.evidence if evidence.correlated_step_id == candidate_step.step_id],+ tier_b=evaluate_tier_b(+ pre_url=_step_url(candidate_step),+ post_url=_step_url(candidate_post_step),+ action_results=_flatten_action_results([candidate_step]),+ pre_page_text=pre_page_text if candidate_step == latest_candidate_step else None,+ post_page_text=post_page_text if candidate_step == latest_candidate_step else None,+ ),+ is_latest=candidate_step == latest_candidate_step,+ )+ for candidate_step, candidate_post_step in candidate_step_pairs+ ]+ verdict = combine(+ tier_a=tier_a,+ candidate_evaluations=candidate_evaluations,+ detected_candidate_step_ids=detected_candidate_step_ids,+ submit_intent_detected=submit_intent_detected,+ browser_path=browser_path,+ cua_run=cua_run,+ coordinate_click=coordinate_click,+ )+ if har_too_large:+ verdict.notes.append("har_too_large")+ return verdict+++def _task_for_candidate_step(tasks: list[Task], candidate_step: Step | None) -> Task:+ if candidate_step is not None:+ for task in tasks:+ if task.task_id == candidate_step.task_id:+ return task+ if not tasks:+ raise ValueError("no tasks available for submission shadow")+ return tasks[-1]+++async def run_submission_signal_shadow(+ *,+ har_data: bytes,+ browser_state: BrowserState,+ last_step: Step,+ task: Task | None = None,+ workflow_run: WorkflowRun | None = None,+ browser_session_id: str | None = None,+) -> None:+ started_at = time.monotonic()+ stage = "load_tasks"+ if task is not None:+ organization_id = task.organization_id+ event_task_id = task.task_id+ event_workflow_run_id = task.workflow_run_id+ elif workflow_run is not None:+ organization_id = workflow_run.organization_id+ event_task_id = last_step.task_id+ event_workflow_run_id = workflow_run.workflow_run_id+ else:+ organization_id = None+ event_task_id = last_step.task_id+ event_workflow_run_id = None+ try:+ context = await _load_context(task=task, workflow_run=workflow_run, last_step=last_step)+ stage = "detect_candidates"+ detection = detect_submit_candidates(context.actions)+ candidate_step_pairs = find_candidate_step_pairs(detection.candidates, context.steps)+ pre_step, post_step = candidate_step_pairs[-1] if candidate_step_pairs else (None, None)+ context_task = _task_for_candidate_step(context.tasks, pre_step)++ stage = "load_run"+ cua_run = await service_utils.is_cua_task(task=context_task)+ stage = "classify_browser_path"+ artifacts = browser_state.browser_artifacts+ browser_path = classify_browser_path(+ browser_session_id=browser_session_id or (workflow_run.browser_session_id if workflow_run else None),+ task_browser_session_id=context_task.browser_session_id,+ remote_browser_session_id=artifacts.remote_browser_session_id,+ task_browser_address=context_task.browser_address+ or (workflow_run.browser_address if workflow_run is not None else None),+ needs_cdp_frame_publisher=artifacts.needs_cdp_frame_publisher,+ browser_type=settings.BROWSER_TYPE,+ )++ candidate_windows = build_candidate_windows(detection.candidates, context.steps)++ stage = "retrieve_page_text"+ pre_page_text = await _retrieve_page_text(pre_step) if pre_step is not None else None+ post_page_text = await _retrieve_page_text(post_step) if post_step is not None else None++ stage = "evaluate"+ verdict = await asyncio.to_thread(+ _evaluate_submission_signal,+ har_data=har_data,+ candidate_windows=candidate_windows,+ candidate_step_pairs=candidate_step_pairs,+ detected_candidate_step_ids=sorted({candidate.step_id for candidate in detection.candidates}),+ latest_candidate_step=pre_step,+ pre_page_text=pre_page_text,+ post_page_text=post_page_text,+ submit_intent_detected=detection.submit_intent_detected,+ browser_path=browser_path,+ cua_run=cua_run,+ coordinate_click=detection.coordinate_click,+ )+ winning_step = next(+ (+ candidate_step+ for candidate_step, _ in candidate_step_pairs+ if candidate_step.step_id == verdict.winning_step_id+ ),+ None,+ )+ event_task = _task_for_candidate_step(context.tasks, winning_step or pre_step)++ LOG.info(+ _SHADOW_EVENT,+ status="ok",+ signal=verdict.signal.value,+ tier_a_hit=bool(verdict.tier_a),+ tier_b_hits=len(verdict.tier_b),+ submit_intent_detected=verdict.submit_intent_detected,+ run_status=context.run_status,+ output_is_none=context.output_is_none,+ har_present=verdict.har_present,+ har_parsed=verdict.har_parsed,+ har_entry_count=verdict.har_entry_count,+ browser_path=verdict.browser_path.value,+ capped=verdict.capped,+ notes=verdict.notes,+ duration_ms=_elapsed_ms(started_at),+ task_id=event_task.task_id,+ workflow_run_id=context.workflow_run_id,+ organization_id=context.organization_id,+ )+ except Exception as exc: # noqa: BLE001 - shadow observability must never affect cleanup+ LOG.warning(+ _SHADOW_EVENT,+ status="error",+ error_stage=stage,+ error_type=type(exc).__name__,+ duration_ms=_elapsed_ms(started_at),+ task_id=event_task_id,+ workflow_run_id=event_workflow_run_id,+ organization_id=organization_id,+ )+++def schedule_submission_signal_shadow(+ *,+ har_data: bytes,+ browser_state: BrowserState,+ last_step: Step,+ task: Task | None = None,+ workflow_run: WorkflowRun | None = None,+ browser_session_id: str | None = None,+) -> asyncio.Task[None] | None:+ if not settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW:+ return None++ try:+ _prune_pending()+ if len(_PENDING_SUBMISSION_SHADOW_TASKS) >= _MAX_PENDING:+ LOG.warning("submission_shadow_task_cap_reached", pending=len(_PENDING_SUBMISSION_SHADOW_TASKS))+ LOG.info(_SHADOW_EVENT, status="skipped", reason="pending_cap")+ return None++ runner = run_submission_signal_shadow(+ har_data=har_data,+ browser_state=browser_state,+ last_step=last_step,+ task=task,+ workflow_run=workflow_run,+ browser_session_id=browser_session_id,+ )+ try:+ scheduled = asyncio.create_task(runner)+ except Exception:+ runner.close()+ raise+ try:+ return _track(scheduled)+ except Exception:+ _PENDING_SUBMISSION_SHADOW_TASKS.discard(scheduled)+ scheduled.cancel()+ raise+ except Exception as exc: # noqa: BLE001 - scheduler must never affect artifact persistence+ try:+ LOG.debug("submission_shadow_schedule_failed", error_type=type(exc).__name__)+ except Exception: # noqa: BLE001 - logging must not break the no-throw boundary+ pass+ return Nonediff --git a/skyvern/forge/sdk/submission/verifier.py b/skyvern/forge/sdk/submission/verifier.pynew file mode 100644index 000000000..b8da1ca13--- /dev/null+++ b/skyvern/forge/sdk/submission/verifier.py@@ -0,0 +1,473 @@+from __future__ import annotations++import hashlib+import json+import re+from collections.abc import Mapping, Sequence+from dataclasses import dataclass+from datetime import UTC, datetime, timedelta+from pathlib import PurePosixPath+from urllib.parse import urlsplit++from skyvern.forge.sdk.copilot.completion_output_grounding import _boundary_delimited_present+from skyvern.forge.sdk.models import Step+from skyvern.forge.sdk.submission.models import (+ BrowserPath,+ CandidateWindow,+ DownloadEvidence,+ NetworkEvidence,+ PageConfirmationEvidence,+ SubmissionSignal,+ SubmissionVerdict,+ SubmitCandidate,+ SubmitCandidateDetection,+ TierAEvaluation,+ TierBEvaluation,+ TierBEvidence,+ UrlTransitionEvidence,+)+from skyvern.webeye.actions.action_types import ActionType+from skyvern.webeye.actions.actions import Action, ClickAction+from skyvern.webeye.actions.responses import ActionResult++_SUBMIT_VERBS = ("place order", "register", "complete", "confirm", "finish", "submit", "apply", "send")+_CONFIRMATION_PHRASES = (+ "confirmation number",+ "successfully submitted",+ "application received",+ "submission received",+ "we have received",+ "thank you",+)+_NETWORK_METHODS = {"POST", "PUT", "PATCH"}+_NETWORK_RESOURCE_TYPES = {"document", "xhr", "fetch"}+_STATIC_MIME_PREFIXES = ("image/", "font/", "text/css")+_STATIC_EXTENSIONS = {+ ".avif",+ ".bmp",+ ".css",+ ".eot",+ ".gif",+ ".ico",+ ".jpeg",+ ".jpg",+ ".js",+ ".map",+ ".otf",+ ".png",+ ".svg",+ ".ttf",+ ".webp",+ ".woff",+ ".woff2",+}+_STEP_WINDOW_SKEW = timedelta(seconds=2)+_STEP_WINDOW_TRAILING = timedelta(seconds=5)+_CONFIRMATION_VALUE_RE = re.compile(r"^[\s:;#-]*([^\s<>,;]+)")+++@dataclass(frozen=True)+class CandidateEvaluation:+ step_id: str+ tier_a: Sequence[NetworkEvidence]+ tier_b: TierBEvaluation+ is_latest: bool = False+++def classify_browser_path(+ *,+ browser_session_id: str | None,+ task_browser_session_id: str | None,+ remote_browser_session_id: str | None,+ task_browser_address: str | None,+ needs_cdp_frame_publisher: bool,+ browser_type: str,+) -> BrowserPath:+ if browser_session_id or task_browser_session_id:+ return BrowserPath.SESSION_ATTACHED+ if remote_browser_session_id:+ return BrowserPath.VENDOR_REUSED+ if task_browser_address or needs_cdp_frame_publisher or browser_type == "cdp-connect":+ return BrowserPath.CDP_CONNECT+ if browser_type in {"chromium-headless", "chromium-headful"}:+ return BrowserPath.SKYVERN_CREATED+ return BrowserPath.UNKNOWN+++def _mapping(value: object) -> Mapping[str, object] | None:+ return value if isinstance(value, Mapping) else None+++def _submit_shaped(element_data: Mapping[str, object]) -> bool:+ tag_name = element_data.get("tagName")+ if not isinstance(tag_name, str):+ return False+ tag_name = tag_name.strip().lower()+ attributes = _mapping(element_data.get("attributes")) or {}+ input_type = attributes.get("type")+ if tag_name in {"button", "input"} and isinstance(input_type, str) and input_type.lower() == "submit":+ return True+ if tag_name not in {"button", "input", "a"}:+ return False++ text_values = [element_data.get("text"), attributes.get("value"), attributes.get("aria-label")]+ return any(+ _boundary_delimited_present(verb, value.strip().lower())+ for value in text_values+ if isinstance(value, str)+ for verb in _SUBMIT_VERBS+ )+++def detect_submit_candidates(actions: Sequence[Action]) -> SubmitCandidateDetection:+ candidates: list[SubmitCandidate] = []+ coordinate_click = False+ for action in actions:+ if action.action_type != ActionType.CLICK:+ continue+ if isinstance(action, ClickAction) and (action.x is not None or action.y is not None):+ coordinate_click = True+ continue+ element_data = _mapping(action.skyvern_element_data)+ if element_data is not None and _submit_shaped(element_data):+ if action.step_id is None:+ continue+ candidates.append(+ SubmitCandidate(+ action_id=action.action_id,+ task_id=action.task_id,+ step_id=action.step_id,+ )+ )+ return SubmitCandidateDetection(candidates=candidates, coordinate_click=coordinate_click)+++def _candidate_actions(+ detection_or_candidates: SubmitCandidateDetection | Sequence[SubmitCandidate],+) -> Sequence[SubmitCandidate]:+ if isinstance(detection_or_candidates, SubmitCandidateDetection):+ return detection_or_candidates.candidates+ return detection_or_candidates+++def build_candidate_windows(+ detection_or_candidates: SubmitCandidateDetection | Sequence[SubmitCandidate],+ steps: Sequence[Step],+) -> list[CandidateWindow]:+ step_by_id = {step.step_id: step for step in steps}+ windows: list[CandidateWindow] = []+ seen_steps: set[str] = set()+ for candidate in _candidate_actions(detection_or_candidates):+ if candidate.step_id is None or candidate.step_id in seen_steps:+ continue+ step = step_by_id.get(candidate.step_id)+ if step is None or (candidate.task_id is not None and candidate.task_id != step.task_id):+ continue+ seen_steps.add(step.step_id)+ windows.append(+ CandidateWindow(+ task_id=step.task_id,+ step_id=step.step_id,+ candidate_action_id=candidate.action_id,+ started_at=step.created_at - _STEP_WINDOW_SKEW,+ ended_at=step.modified_at + _STEP_WINDOW_TRAILING,+ )+ )+ return sorted(windows, key=lambda window: window.started_at)+++def find_candidate_step_pairs(+ detection_or_candidates: SubmitCandidateDetection | Sequence[SubmitCandidate],+ steps: Sequence[Step],+) -> list[tuple[Step, Step | None]]:+ ordered = sorted(steps, key=lambda step: (step.task_id, step.order, step.retry_index, step.created_at))+ positions = {step.step_id: index for index, step in enumerate(ordered)}+ pairs: list[tuple[Step, Step | None]] = []+ seen_steps: set[str] = set()+ for candidate in _candidate_actions(detection_or_candidates):+ if candidate.step_id is None or candidate.step_id in seen_steps or candidate.step_id not in positions:+ continue+ pre_step = ordered[positions[candidate.step_id]]+ if candidate.task_id is not None and candidate.task_id != pre_step.task_id:+ continue+ seen_steps.add(candidate.step_id)+ index = positions[pre_step.step_id]+ post_step = ordered[index + 1] if index + 1 < len(ordered) else None+ if post_step is not None and post_step.task_id != pre_step.task_id:+ post_step = None+ pairs.append((pre_step, post_step))+ return sorted(pairs, key=lambda pair: pair[0].modified_at)+++def find_latest_candidate_step_pair(+ detection_or_candidates: SubmitCandidateDetection | Sequence[SubmitCandidate],+ steps: Sequence[Step],+) -> tuple[Step | None, Step | None]:+ pairs = find_candidate_step_pairs(detection_or_candidates, steps)+ if not pairs:+ return None, None+ return pairs[-1]+++def _utc(value: datetime) -> datetime:+ if value.tzinfo is None:+ return value.replace(tzinfo=UTC)+ return value.astimezone(UTC)+++def _parse_har_time(value: str) -> datetime | None:+ try:+ return _utc(datetime.fromisoformat(value.replace("Z", "+00:00")))+ except ValueError:+ return None+++def _origin(url: str) -> str | None:+ try:+ parsed = urlsplit(url)+ if not parsed.scheme or parsed.hostname is None:+ return None+ port = f":{parsed.port}" if parsed.port is not None else ""+ return f"{parsed.scheme.lower()}://{parsed.hostname.lower()}{port}"+ except ValueError:+ return None+++def _path(url: str) -> str | None:+ try:+ parsed = urlsplit(url)+ except ValueError:+ return None+ if not parsed.scheme or parsed.hostname is None:+ return None+ return parsed.path or "/"+++def _content_type(response: Mapping[str, object]) -> str:+ content = _mapping(response.get("content"))+ mime_type = content.get("mimeType") if content is not None else None+ if isinstance(mime_type, str):+ return mime_type.lower()+ headers = response.get("headers")+ if isinstance(headers, list):+ for header in headers:+ header_data = _mapping(header)+ if header_data is None:+ continue+ name = header_data.get("name")+ value = header_data.get("value")+ if isinstance(name, str) and name.lower() == "content-type" and isinstance(value, str):+ return value.lower()+ return ""+++def _is_static(entry: Mapping[str, object], response: Mapping[str, object], url: str) -> bool:+ resource_type = entry.get("_resourceType")+ if isinstance(resource_type, str) and resource_type:+ return resource_type.lower() not in _NETWORK_RESOURCE_TYPES+ content_type = _content_type(response)+ if any(content_type.startswith(prefix) for prefix in _STATIC_MIME_PREFIXES):+ return True+ return PurePosixPath(urlsplit(url).path.lower()).suffix in _STATIC_EXTENSIONS+++def evaluate_tier_a(har_bytes: bytes, candidate_windows: Sequence[CandidateWindow]) -> TierAEvaluation:+ har_present = bool(har_bytes)+ if not har_present:+ return TierAEvaluation(evidence=[], har_present=False, har_parsed=False, har_entry_count=0)+ try:+ payload = json.loads(har_bytes)+ log = _mapping(payload.get("log")) if isinstance(payload, Mapping) else None+ entries = log.get("entries") if log is not None else None+ except (UnicodeDecodeError, json.JSONDecodeError, RecursionError):+ entries = None+ if not isinstance(entries, list):+ return TierAEvaluation(evidence=[], har_present=True, har_parsed=False, har_entry_count=0)++ evidence: list[NetworkEvidence] = []+ ambiguous_entry_count = 0+ for entry_value in entries:+ entry = _mapping(entry_value)+ request = _mapping(entry.get("request")) if entry is not None else None+ response = _mapping(entry.get("response")) if entry is not None else None+ started_value = entry.get("startedDateTime") if entry is not None else None+ if entry is None or request is None or response is None or not isinstance(started_value, str):+ return TierAEvaluation(evidence=[], har_present=True, har_parsed=False, har_entry_count=len(entries))+ method = request.get("method")+ url = request.get("url")+ status = response.get("status")+ started_at = _parse_har_time(started_value)+ origin = _origin(url) if isinstance(url, str) else None+ if (+ not isinstance(method, str)+ or not isinstance(url, str)+ or not isinstance(status, int)+ or isinstance(status, bool)+ or started_at is None+ or origin is None+ ):+ return TierAEvaluation(evidence=[], har_present=True, har_parsed=False, har_entry_count=len(entries))+ if method.upper() not in _NETWORK_METHODS or not 200 <= status < 400 or _is_static(entry, response, url):+ continue+ matching_windows = [+ window for window in candidate_windows if _utc(window.started_at) <= started_at <= _utc(window.ended_at)+ ]+ if len(matching_windows) > 1:+ ambiguous_entry_count += 1+ continue+ if len(matching_windows) == 1:+ evidence.append(+ NetworkEvidence(+ origin=origin,+ method=method.upper(),+ status=status,+ started_at=started_at,+ correlated_step_id=matching_windows[0].step_id,+ )+ )+ return TierAEvaluation(+ evidence=evidence,+ har_present=True,+ har_parsed=True,+ har_entry_count=len(entries),+ ambiguous_entry_count=ambiguous_entry_count,+ )+++def _confirmation_value(phrase: str, normalized_post_text: str) -> str:+ if phrase != "confirmation number":+ return phrase+ index = normalized_post_text.find(phrase)+ match = _CONFIRMATION_VALUE_RE.match(normalized_post_text[index + len(phrase) :])+ return match.group(1) if match else phrase+++def evaluate_tier_b(+ *,+ pre_url: str | None,+ post_url: str | None,+ action_results: Sequence[ActionResult],+ pre_page_text: str | None,+ post_page_text: str | None,+) -> TierBEvaluation:+ evidence: list[TierBEvidence] = []+ if pre_url is not None and post_url is not None:+ from_origin = _origin(pre_url)+ to_origin = _origin(post_url)+ from_path = _path(pre_url)+ to_path = _path(post_url)+ if from_origin is not None and to_origin is not None and from_path is not None and to_path is not None:+ path_changed = from_path != to_path+ if from_origin != to_origin or path_changed:+ evidence.append(+ UrlTransitionEvidence(+ from_origin=from_origin,+ to_origin=to_origin,+ path_changed=path_changed,+ )+ )++ file_count = sum(+ max(len(result.downloaded_files or []), 1 if result.download_triggered else 0) for result in action_results+ )+ if file_count:+ evidence.append(DownloadEvidence(file_count=file_count))++ page_evaluated = False+ if pre_page_text is not None and post_page_text is not None:+ page_evaluated = True+ normalized_pre = pre_page_text.casefold()+ normalized_post = post_page_text.casefold()+ for phrase in _CONFIRMATION_PHRASES:+ if _boundary_delimited_present(phrase, normalized_post) and not _boundary_delimited_present(+ phrase, normalized_pre+ ):+ matched_value = _confirmation_value(phrase, normalized_post)+ evidence.append(+ PageConfirmationEvidence(+ phrase=phrase,+ value_sha256=hashlib.sha256(matched_value.encode()).hexdigest(),+ absent_pre_submit=True,+ )+ )+ break+ return TierBEvaluation(evidence=evidence, page_confirmation_evaluated=page_evaluated)+++def combine(+ *,+ tier_a: TierAEvaluation,+ candidate_evaluations: Sequence[CandidateEvaluation],+ detected_candidate_step_ids: Sequence[str],+ submit_intent_detected: bool,+ browser_path: BrowserPath,+ cua_run: bool = False,+ coordinate_click: bool = False,+) -> SubmissionVerdict:+ tier_a_allowed = browser_path == BrowserPath.SKYVERN_CREATED++ def candidate_strength(candidate: CandidateEvaluation) -> int:+ has_tier_a = tier_a_allowed and bool(candidate.tier_a)+ has_tier_b = bool(candidate.tier_b.evidence)+ if has_tier_a and has_tier_b:+ return 2+ return int(has_tier_a or has_tier_b)++ winning_candidate = max(+ candidate_evaluations,+ key=lambda candidate: (candidate_strength(candidate), candidate.is_latest),+ default=None,+ )+ tier_a_evidence = list(winning_candidate.tier_a) if winning_candidate is not None and tier_a_allowed else []+ tier_b_evidence = list(winning_candidate.tier_b.evidence) if winning_candidate is not None else []+ winning_strength = candidate_strength(winning_candidate) if winning_candidate is not None else 0+ capped = False+ notes: list[str] = []+ if tier_a.ambiguous_entry_count:+ notes.append(f"ambiguous_entries:{tier_a.ambiguous_entry_count}")++ if cua_run or coordinate_click:+ signal = SubmissionSignal.NOT_EVALUATED+ notes.append("cua_or_coordinate_click")+ elif not submit_intent_detected:+ signal = SubmissionSignal.NOT_EVALUATED+ notes.append("submit_intent_not_detected")+ elif browser_path != BrowserPath.SKYVERN_CREATED:+ capped = True+ signal = SubmissionSignal.SUBMITTED_LIKELY if winning_strength else SubmissionSignal.NOT_EVALUATED+ notes.append("tier_b_only_browser_path")+ elif winning_strength == 2:+ signal = SubmissionSignal.SUBMITTED_VERIFIED+ elif winning_strength == 1:+ signal = SubmissionSignal.SUBMITTED_LIKELY+ elif (+ tier_a.har_present+ and tier_a.har_parsed+ and tier_a.har_entry_count > 0+ # An ambiguous entry is a discarded, possibly real submission POST — absence is not affirmative.+ and tier_a.ambiguous_entry_count == 0+ and len(candidate_evaluations) > 0+ # A detected candidate whose step could not be mapped (actions.step_id has no FK) is unknown+ # evidence — evaluations must cover every detected candidate step.+ and set(detected_candidate_step_ids) <= {candidate.step_id for candidate in candidate_evaluations}+ # Every candidate's page-confirmation check must have actually run, not just the latest one's.+ and all(candidate.tier_b.page_confirmation_evaluated for candidate in candidate_evaluations)+ and all(not candidate.tier_a and not candidate.tier_b.evidence for candidate in candidate_evaluations)+ ):+ signal = SubmissionSignal.NOT_SUBMITTED+ else:+ signal = SubmissionSignal.NOT_EVALUATED++ return SubmissionVerdict(+ signal=signal,+ tier_a=tier_a_evidence,+ tier_b=tier_b_evidence,+ submit_intent_detected=submit_intent_detected,+ har_present=tier_a.har_present,+ har_parsed=tier_a.har_parsed,+ har_entry_count=tier_a.har_entry_count,+ browser_path=browser_path,+ winning_step_id=winning_candidate.step_id if winning_candidate is not None else None,+ capped=capped,+ notes=notes,+ )diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 5e2512174..9adaf141c 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -76,6 +76,7 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import ()from skyvern.forge.sdk.schemas.tasks import Task, TaskStatusfrom skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock, WorkflowRunTimeline, WorkflowRunTimelineType+from skyvern.forge.sdk.submission import shadow as submission_shadowfrom skyvern.forge.sdk.trace import tracedfrom skyvern.forge.sdk.workflow.browser_profile_key import (build_browser_profile_key_digest,@@ -7421,6 +7422,13 @@ class WorkflowService:workflow_run_id=workflow_run.workflow_run_id,browser_state=browser_state,)+ if settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW:+ submission_shadow.schedule_submission_signal_shadow(+ har_data=har_data,+ browser_state=browser_state,+ last_step=last_step,+ workflow_run=workflow_run,+ )LOG.debug("Persisting har data", har_size=len(har_data))if har_data:await app.ARTIFACT_MANAGER.create_artifact(@@ -7503,6 +7511,13 @@ class WorkflowService:workflow_run_id=workflow_run.workflow_run_id,browser_state=browser_state,)+ if settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW:+ submission_shadow.schedule_submission_signal_shadow(+ har_data=har_data,+ browser_state=browser_state,+ last_step=last_step,+ workflow_run=workflow_run,+ )LOG.debug("Persisting har data (bundled)", har_size=len(har_data))if har_data:task_archive_entries["har.har"] = (ArtifactType.HAR, har_data)diff --git a/skyvern/services/service_utils.py b/skyvern/services/service_utils.pyindex d97da8b32..fc845eb72 100644--- a/skyvern/services/service_utils.py+++ b/skyvern/services/service_utils.py@@ -1,4 +1,5 @@from skyvern.forge import app+from skyvern.forge.sdk.db.exceptions import NotFoundErrorfrom skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.schemas.runs import CUA_ENGINES, CUA_RUN_TYPES@@ -11,11 +12,15 @@ async def is_cua_task(if task.workflow_run_id:# it's a task based block, should look up the block run to see if it's a CUA task- block = await app.DATABASE.observer.get_workflow_run_block_by_task_id(- task_id=task.task_id,- organization_id=task.organization_id,- )- if block.engine is not None and block.engine in CUA_ENGINES:+ try:+ block = await app.DATABASE.observer.get_workflow_run_block_by_task_id(+ task_id=task.task_id,+ organization_id=task.organization_id,+ )+ except NotFoundError:+ # a missing block row means "unknown", not CUA — fall through to the run-type check+ block = None+ if block is not None and block.engine is not None and block.engine in CUA_ENGINES:return Truerun = await app.DATABASE.tasks.get_run(diff --git a/tests/unit/submission/test_shadow.py b/tests/unit/submission/test_shadow.pynew file mode 100644index 000000000..8c5dc903c--- /dev/null+++ b/tests/unit/submission/test_shadow.py@@ -0,0 +1,774 @@+from __future__ import annotations++import asyncio+import json+from datetime import UTC, datetime, timedelta+from types import SimpleNamespace+from typing import Any, cast+from unittest.mock import AsyncMock, Mock++import pytest+from structlog.testing import capture_logs++from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType+from skyvern.forge.sdk.models import Step, StepStatus+from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus+from skyvern.forge.sdk.submission import shadow+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus+from skyvern.schemas.run_enums import RunEngine, RunType+from skyvern.schemas.steps import AgentStepOutput, BrowserMetadata+from skyvern.webeye.actions.actions import ClickAction+from skyvern.webeye.actions.responses import ActionResult+from skyvern.webeye.browser_artifacts import BrowserArtifacts+from skyvern.webeye.browser_state import BrowserState++NOW = datetime(2026, 7, 9, 12, 0, 0)+++def _task(task_id: str = "task-1", *, workflow_run_id: str | None = None) -> Task:+ return Task(+ task_id=task_id,+ status=TaskStatus.completed,+ created_at=NOW,+ modified_at=NOW,+ organization_id="org-1",+ workflow_run_id=workflow_run_id,+ url="https://example.com/form",+ )+++def _submit_action(*, task_id: str = "task-1", step_id: str = "step-1") -> ClickAction:+ return ClickAction(+ element_id="submit-button",+ task_id=task_id,+ step_id=step_id,+ created_at=NOW + timedelta(seconds=2),+ skyvern_element_data={+ "tagName": "button",+ "attributes": {"type": "submit"},+ "text": "Submit",+ },+ )+++def _step(+ step_id: str,+ order: int,+ *,+ task_id: str = "task-1",+ url: str = "https://example.com/form",+ action: ClickAction | None = None,+) -> Step:+ pairs = [] if action is None else [(action, [ActionResult(success=True)])]+ return Step(+ created_at=NOW + timedelta(seconds=order * 10),+ modified_at=NOW + timedelta(seconds=order * 10 + 5),+ task_id=task_id,+ step_id=step_id,+ status=StepStatus.completed,+ output=AgentStepOutput(+ actions_and_results=pairs,+ browser_metadata=BrowserMetadata(website_url=url),+ ),+ order=order,+ is_last=False,+ organization_id="org-1",+ )+++def _workflow_run() -> WorkflowRun:+ return WorkflowRun(+ workflow_run_id="workflow-run-1",+ workflow_id="workflow-1",+ workflow_permanent_id="wpid-1",+ organization_id="org-1",+ status=WorkflowRunStatus.completed,+ created_at=NOW,+ modified_at=NOW,+ )+++def _browser_state() -> BrowserState:+ return cast(+ BrowserState,+ SimpleNamespace(+ browser_artifacts=BrowserArtifacts(),+ ),+ )+++def _har(*, method: str = "POST", status: int = 200, offset_seconds: int = 12) -> bytes:+ return json.dumps(+ {+ "log": {+ "entries": [+ {+ "startedDateTime": (NOW + timedelta(seconds=offset_seconds)).replace(tzinfo=UTC).isoformat(),+ "request": {"method": method, "url": "https://example.com/submit?token=private"},+ "response": {"status": status, "content": {"mimeType": "application/json"}},+ "_resourceType": "xhr",+ }+ ]+ }+ }+ ).encode()+++def _artifact(artifact_id: str, step_id: str, artifact_type: ArtifactType) -> Artifact:+ return Artifact(+ artifact_id=artifact_id,+ artifact_type=artifact_type,+ uri=f"s3://artifacts/{artifact_id}",+ task_id="task-1",+ step_id=step_id,+ organization_id="org-1",+ created_at=NOW,+ modified_at=NOW,+ )+++def _install_task_dependencies(+ monkeypatch: pytest.MonkeyPatch,+ *,+ steps: list[Step],+ actions: list[ClickAction],+ artifact_payloads: dict[str, bytes | None],+ include_visible_artifacts: bool = True,+) -> tuple[SimpleNamespace, AsyncMock]:+ visible_by_step = {+ step.step_id: _artifact(f"visible-{step.step_id}", step.step_id, ArtifactType.VISIBLE_ELEMENTS_TREE_IN_PROMPT)+ for step in steps+ }+ html_by_step = {+ step.step_id: _artifact(f"html-{step.step_id}", step.step_id, ArtifactType.HTML_SCRAPE) for step in steps+ }++ async def get_artifacts_for_task_step(+ task_id: str,+ step_id: str,+ organization_id: str | None = None,+ ) -> list[Artifact]:+ assert task_id == "task-1"+ assert organization_id == "org-1"+ if include_visible_artifacts:+ return [html_by_step[step_id], visible_by_step[step_id]]+ return [html_by_step[step_id]]++ tasks_repo = SimpleNamespace(+ get_task_steps=AsyncMock(return_value=steps),+ get_task_actions_hydrated=AsyncMock(return_value=actions),+ get_run=AsyncMock(return_value=SimpleNamespace(task_run_type=RunType.task_v1)),+ )+ artifacts_repo = SimpleNamespace(get_artifacts_for_task_step=AsyncMock(side_effect=get_artifacts_for_task_step))+ observer_repo = SimpleNamespace(get_workflow_run_block_by_task_id=AsyncMock())+ database = SimpleNamespace(tasks=tasks_repo, artifacts=artifacts_repo, observer=observer_repo)+ retrieve_artifact = AsyncMock(side_effect=lambda artifact: artifact_payloads.get(artifact.artifact_id))+ monkeypatch.setattr(shadow.app, "DATABASE", database)+ monkeypatch.setattr(shadow.app, "ARTIFACT_MANAGER", SimpleNamespace(retrieve_artifact=retrieve_artifact))+ return database, retrieve_artifact++[email protected](autouse=True)+def _clear_pending_tasks() -> None:+ shadow._PENDING_SUBMISSION_SHADOW_TASKS.clear()+ yield+ shadow._PENDING_SUBMISSION_SHADOW_TASKS.clear()+++def test_flag_off_returns_before_pruning_or_logging(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", False)+ monkeypatch.setattr(shadow, "_prune_pending", Mock(side_effect=AssertionError("must not prune")))+ task = _task()+ last_step = _step("step-1", 1)++ with capture_logs() as logs:+ scheduled = shadow.schedule_submission_signal_shadow(+ har_data=b"",+ browser_state=_browser_state(),+ last_step=last_step,+ task=task,+ )++ assert scheduled is None+ assert shadow._PENDING_SUBMISSION_SHADOW_TASKS == set()+ assert logs == []+++def test_pending_task_cap_drops_work_and_warns(monkeypatch: pytest.MonkeyPatch) -> None:+ class _PendingMarker:+ def done(self) -> bool:+ return False++ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", True)+ assert shadow._MAX_PENDING == 20+ pending: set[Any] = {_PendingMarker() for _ in range(shadow._MAX_PENDING)}+ monkeypatch.setattr(shadow, "_PENDING_SUBMISSION_SHADOW_TASKS", pending)++ with capture_logs() as logs:+ scheduled = shadow.schedule_submission_signal_shadow(+ har_data=b"",+ browser_state=_browser_state(),+ last_step=_step("step-1", 1),+ task=_task(),+ )++ assert scheduled is None+ assert len(shadow._PENDING_SUBMISSION_SHADOW_TASKS) == shadow._MAX_PENDING+ assert logs == [+ {+ "event": "submission_shadow_task_cap_reached",+ "log_level": "warning",+ "pending": shadow._MAX_PENDING,+ },+ {+ "event": "submission_signal_shadow",+ "log_level": "info",+ "reason": "pending_cap",+ "status": "skipped",+ },+ ]+++def test_scheduler_swallows_setup_failures(monkeypatch: pytest.MonkeyPatch) -> None:+ log = SimpleNamespace(debug=Mock())+ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", True)+ monkeypatch.setattr(shadow, "_prune_pending", Mock(side_effect=RuntimeError("private scheduler detail")))+ monkeypatch.setattr(shadow, "LOG", log)++ scheduled = shadow.schedule_submission_signal_shadow(+ har_data=b"",+ browser_state=_browser_state(),+ last_step=_step("step-1", 1),+ task=_task(),+ )++ assert scheduled is None+ log.debug.assert_called_once_with("submission_shadow_schedule_failed", error_type="RuntimeError")+++async def test_scheduler_prunes_done_tasks_before_applying_cap(monkeypatch: pytest.MonkeyPatch) -> None:+ class _DoneMarker:+ def done(self) -> bool:+ return True++ async def fake_runner(**_kwargs: object) -> None:+ return None++ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", True)+ monkeypatch.setattr(shadow, "run_submission_signal_shadow", fake_runner)+ pending: set[Any] = {_DoneMarker() for _ in range(shadow._MAX_PENDING)}+ monkeypatch.setattr(shadow, "_PENDING_SUBMISSION_SHADOW_TASKS", pending)++ with capture_logs() as logs:+ scheduled = shadow.schedule_submission_signal_shadow(+ har_data=b"",+ browser_state=_browser_state(),+ last_step=_step("step-1", 1),+ task=_task(),+ )++ assert scheduled is not None+ await scheduled+ await asyncio.sleep(0)+ assert shadow._PENDING_SUBMISSION_SHADOW_TASKS == set()+ assert logs == []+++async def test_scheduler_returns_and_tracks_background_task(monkeypatch: pytest.MonkeyPatch) -> None:+ started = asyncio.Event()+ release = asyncio.Event()++ async def fake_runner(**_kwargs: object) -> None:+ started.set()+ await release.wait()++ monkeypatch.setattr(shadow.settings, "SKYVERN_SUBMISSION_SIGNAL_SHADOW", True)+ monkeypatch.setattr(shadow, "run_submission_signal_shadow", fake_runner)++ scheduled = shadow.schedule_submission_signal_shadow(+ har_data=b"",+ browser_state=_browser_state(),+ last_step=_step("step-1", 1),+ task=_task(),+ )++ assert scheduled is not None+ await started.wait()+ assert scheduled in shadow._PENDING_SUBMISSION_SHADOW_TASKS+ release.set()+ await scheduled+ await asyncio.sleep(0)+ assert scheduled not in shadow._PENDING_SUBMISSION_SHADOW_TASKS+++ ("include_visible_artifacts", "artifact_prefix"),+ [(True, "visible"), (False, "html")],+)+async def test_runner_uses_hydrated_actions_next_step_text_and_emits_one_aggregate_event(+ monkeypatch: pytest.MonkeyPatch,+ include_visible_artifacts: bool,+ artifact_prefix: str,+) -> None:+ action = _submit_action()+ pre_step = _step("step-1", 1, action=action)+ post_step = _step("step-2", 2)+ database, retrieve_artifact = _install_task_dependencies(+ monkeypatch,+ steps=[pre_step, post_step],+ actions=[action],+ artifact_payloads={+ "visible-step-1": b"Review the details before sending.",+ "visible-step-2": b"Thank you. Your request is complete.",+ "html-step-1": b"Review the details before sending.",+ "html-step-2": b"Thank you. Your request is complete.",+ },+ include_visible_artifacts=include_visible_artifacts,+ )+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")+ to_thread = AsyncMock(side_effect=lambda function, *args, **kwargs: function(*args, **kwargs))+ monkeypatch.setattr(shadow.asyncio, "to_thread", to_thread)++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(),+ browser_state=_browser_state(),+ last_step=post_step,+ task=_task(),+ )++ events = [record for record in logs if record["event"] == "submission_signal_shadow"]+ assert len(events) == 1+ assert events[0]["status"] == "ok"+ assert events[0]["signal"] == "submitted_verified"+ assert events[0]["tier_a_hit"] is True+ assert events[0]["tier_b_hits"] >= 1+ assert "evidence" not in events[0]+ to_thread.assert_awaited_once()+ database.tasks.get_task_actions_hydrated.assert_awaited_once_with("task-1", organization_id="org-1")+ database.tasks.get_run.assert_awaited_once_with(run_id="task-1", organization_id="org-1")+ assert [call.args[0].artifact_id for call in retrieve_artifact.await_args_list] == [+ f"{artifact_prefix}-step-1",+ f"{artifact_prefix}-step-2",+ ]+++async def test_runner_skips_oversized_har_without_allowing_not_submitted(monkeypatch: pytest.MonkeyPatch) -> None:+ action = _submit_action()+ pre_step = _step("step-1", 1, action=action)+ post_step = _step("step-2", 2)+ _install_task_dependencies(+ monkeypatch,+ steps=[pre_step, post_step],+ actions=[action],+ artifact_payloads={+ "visible-step-1": b"Review the details before sending.",+ "visible-step-2": b"The form remains ready.",+ },+ )+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")+ monkeypatch.setattr(shadow, "_MAX_HAR_BYTES", 10)++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=b"x" * 11,+ browser_state=_browser_state(),+ last_step=post_step,+ task=_task(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["signal"] == "not_evaluated"+ assert event["har_present"] is True+ assert event["har_parsed"] is False+ assert "har_too_large" in event["notes"]+++async def test_unmapped_candidate_step_cannot_produce_not_submitted(monkeypatch: pytest.MonkeyPatch) -> None:+ mapped_action = _submit_action()+ ghost_action = _submit_action(step_id="step-ghost")+ pre_step = _step("step-1", 1, action=mapped_action)+ post_step = _step("step-2", 2)+ _install_task_dependencies(+ monkeypatch,+ steps=[pre_step, post_step],+ actions=[mapped_action, ghost_action],+ artifact_payloads={+ "visible-step-1": b"Review the details before sending.",+ "visible-step-2": b"The form remains ready.",+ },+ )+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(method="GET"),+ browser_state=_browser_state(),+ last_step=post_step,+ task=_task(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["har_present"] is True+ assert event["har_parsed"] is True+ assert event["signal"] == "not_evaluated"+++async def test_zip_page_artifacts_cannot_produce_not_submitted(monkeypatch: pytest.MonkeyPatch) -> None:+ action = _submit_action()+ pre_step = _step("step-1", 1, action=action)+ post_step = _step("step-2", 2)+ _, retrieve_artifact = _install_task_dependencies(+ monkeypatch,+ steps=[pre_step, post_step],+ actions=[action],+ artifact_payloads={+ "visible-step-1": b"PK\x03\x04pre-archive",+ "visible-step-2": b"PK\x03\x04post-archive",+ },+ )+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(method="GET"),+ browser_state=_browser_state(),+ last_step=post_step,+ task=_task(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["status"] == "ok"+ assert event["signal"] == "not_evaluated"+ assert event["signal"] != "not_submitted"+ assert retrieve_artifact.await_count == 2+++async def test_unrelated_step_download_does_not_count_as_candidate_evidence(monkeypatch: pytest.MonkeyPatch) -> None:+ unrelated_step = _step("step-0", 0)+ assert unrelated_step.output is not None+ unrelated_step.output.action_results = [ActionResult(success=True, download_triggered=True)]+ action = _submit_action()+ pre_step = _step("step-1", 1, action=action)+ post_step = _step("step-2", 2)+ _install_task_dependencies(+ monkeypatch,+ steps=[unrelated_step, pre_step, post_step],+ actions=[action],+ artifact_payloads={+ "visible-step-1": b"Review the details before sending.",+ "visible-step-2": b"The form remains ready.",+ },+ )+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(method="GET"),+ browser_state=_browser_state(),+ last_step=post_step,+ task=_task(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["status"] == "ok"+ assert event["signal"] == "not_submitted"+ assert event["tier_b_hits"] == 0+++async def test_latest_unpaired_candidate_cannot_reuse_older_page_completeness(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ older_action = _submit_action(step_id="step-1")+ latest_action = _submit_action(step_id="step-3")+ older_step = _step("step-1", 1, action=older_action)+ older_post_step = _step("step-2", 2)+ latest_step = _step("step-3", 3, action=latest_action)+ _, retrieve_artifact = _install_task_dependencies(+ monkeypatch,+ steps=[older_step, older_post_step, latest_step],+ actions=[older_action, latest_action],+ artifact_payloads={+ "visible-step-1": b"Review the details.",+ "visible-step-2": b"The form remains ready.",+ "visible-step-3": b"Review the latest details.",+ },+ )+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(method="GET"),+ browser_state=_browser_state(),+ last_step=latest_step,+ task=_task(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["status"] == "ok"+ assert event["signal"] == "not_evaluated"+ assert [call.args[0].step_id for call in retrieve_artifact.await_args_list] == ["step-3"]+++async def test_runner_does_not_cross_pair_older_network_with_latest_transition(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ older_action = _submit_action(step_id="step-1")+ latest_action = _submit_action(step_id="step-3")+ older_step = _step("step-1", 1, action=older_action)+ older_post_step = _step("step-2", 2)+ latest_step = _step("step-3", 3, action=latest_action)+ latest_post_step = _step("step-4", 4, url="https://example.com/complete")+ _install_task_dependencies(+ monkeypatch,+ steps=[older_step, older_post_step, latest_step, latest_post_step],+ actions=[older_action, latest_action],+ artifact_payloads={+ "visible-step-3": b"Review the latest details.",+ "visible-step-4": b"The form remains ready.",+ },+ )+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(offset_seconds=12),+ browser_state=_browser_state(),+ last_step=latest_post_step,+ task=_task(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["status"] == "ok"+ assert event["signal"] == "submitted_likely"+ assert event["tier_a_hit"] is False+ assert event["tier_b_hits"] == 1+++async def test_runner_can_select_an_older_verified_candidate_without_extra_page_fetches(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ older_action = _submit_action(step_id="step-1")+ latest_action = _submit_action(step_id="step-3")+ older_step = _step("step-1", 1, action=older_action)+ older_post_step = _step("step-2", 2, url="https://example.com/complete")+ latest_step = _step("step-3", 3, action=latest_action)+ _, retrieve_artifact = _install_task_dependencies(+ monkeypatch,+ steps=[older_step, older_post_step, latest_step],+ actions=[older_action, latest_action],+ artifact_payloads={"visible-step-3": b"Review the latest details."},+ )+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(offset_seconds=12),+ browser_state=_browser_state(),+ last_step=latest_step,+ task=_task(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["status"] == "ok"+ assert event["signal"] == "submitted_verified"+ assert event["tier_a_hit"] is True+ assert event["tier_b_hits"] == 1+ assert [call.args[0].step_id for call in retrieve_artifact.await_args_list] == ["step-3"]+++ ("run_type", "include_coordinate_click"),+ [(RunType.openai_cua, False), (RunType.task_v1, True)],+)+async def test_cua_run_and_coordinate_click_overrides_are_applied_end_to_end(+ monkeypatch: pytest.MonkeyPatch,+ run_type: RunType,+ include_coordinate_click: bool,+) -> None:+ action = _submit_action()+ actions = [action]+ if include_coordinate_click:+ actions.append(ClickAction(element_id="", step_id="step-1", x=10, y=20))+ pre_step = _step("step-1", 1, action=action)+ post_step = _step("step-2", 2)+ database, _ = _install_task_dependencies(+ monkeypatch,+ steps=[pre_step, post_step],+ actions=actions,+ artifact_payloads={+ "visible-step-1": b"Review the details.",+ "visible-step-2": b"Thank you. Your request is complete.",+ },+ )+ database.tasks.get_run.return_value = SimpleNamespace(task_run_type=run_type)+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(),+ browser_state=_browser_state(),+ last_step=post_step,+ task=_task(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["status"] == "ok"+ assert event["signal"] == "not_evaluated"+ database.tasks.get_run.assert_awaited_once_with(run_id="task-1", organization_id="org-1")+++async def test_workflow_uses_all_tasks_and_block_engine_for_cua_detection(monkeypatch: pytest.MonkeyPatch) -> None:+ first_task = _task("task-a", workflow_run_id="workflow-run-1")+ candidate_task = _task("task-1", workflow_run_id="workflow-run-1")+ action = _submit_action()+ first_step = _step("step-a", 1, task_id="task-a")+ pre_step = _step("step-1", 2, action=action)+ post_step = _step("step-2", 3)++ async def get_steps(task_id: str, organization_id: str) -> list[Step]:+ assert organization_id == "org-1"+ return [first_step] if task_id == "task-a" else [pre_step, post_step]++ async def get_actions(task_id: str, organization_id: str | None = None) -> list[ClickAction]:+ assert organization_id == "org-1"+ return [] if task_id == "task-a" else [action]++ tasks_repo = SimpleNamespace(+ get_tasks_by_workflow_run_id=AsyncMock(return_value=[first_task, candidate_task]),+ get_task_steps=AsyncMock(side_effect=get_steps),+ get_task_actions_hydrated=AsyncMock(side_effect=get_actions),+ get_run=AsyncMock(return_value=SimpleNamespace(task_run_type=RunType.task_v1)),+ )+ artifacts_repo = SimpleNamespace(get_artifacts_for_task_step=AsyncMock(return_value=[]))+ observer_repo = SimpleNamespace(+ get_workflow_run_block_by_task_id=AsyncMock(return_value=SimpleNamespace(engine=RunEngine.openai_cua))+ )+ workflow_runs_repo = SimpleNamespace(+ get_workflow_run_output_parameters=AsyncMock(return_value=[SimpleNamespace(value="workflow-output")])+ )+ monkeypatch.setattr(+ shadow.app,+ "DATABASE",+ SimpleNamespace(+ tasks=tasks_repo,+ artifacts=artifacts_repo,+ observer=observer_repo,+ workflow_runs=workflow_runs_repo,+ ),+ )+ monkeypatch.setattr(shadow.app, "ARTIFACT_MANAGER", SimpleNamespace(retrieve_artifact=AsyncMock()))+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(),+ browser_state=_browser_state(),+ last_step=post_step,+ workflow_run=_workflow_run(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["status"] == "ok"+ assert event["signal"] == "not_evaluated"+ assert event["task_id"] == "task-1"+ assert event["workflow_run_id"] == "workflow-run-1"+ assert event["output_is_none"] is False+ tasks_repo.get_tasks_by_workflow_run_id.assert_awaited_once_with(workflow_run_id="workflow-run-1")+ assert tasks_repo.get_task_steps.await_count == 2+ assert tasks_repo.get_task_actions_hydrated.await_count == 2+ observer_repo.get_workflow_run_block_by_task_id.assert_awaited_once_with(+ task_id="task-1",+ organization_id="org-1",+ )+ workflow_runs_repo.get_workflow_run_output_parameters.assert_awaited_once_with(workflow_run_id="workflow-run-1")+++async def test_workflow_event_task_follows_the_winning_candidate(monkeypatch: pytest.MonkeyPatch) -> None:+ older_task = _task("task-a", workflow_run_id="workflow-run-1")+ latest_task = _task("task-b", workflow_run_id="workflow-run-1")+ older_action = _submit_action(task_id="task-a", step_id="step-a1")+ latest_action = _submit_action(task_id="task-b", step_id="step-b1")+ older_step = _step("step-a1", 1, task_id="task-a", action=older_action)+ older_post_step = _step("step-a2", 2, task_id="task-a", url="https://example.com/complete")+ latest_step = _step("step-b1", 3, task_id="task-b", action=latest_action)++ async def get_steps(task_id: str, organization_id: str) -> list[Step]:+ assert organization_id == "org-1"+ return [older_step, older_post_step] if task_id == "task-a" else [latest_step]++ async def get_actions(task_id: str, organization_id: str | None = None) -> list[ClickAction]:+ assert organization_id == "org-1"+ return [older_action] if task_id == "task-a" else [latest_action]++ tasks_repo = SimpleNamespace(+ get_tasks_by_workflow_run_id=AsyncMock(return_value=[older_task, latest_task]),+ get_task_steps=AsyncMock(side_effect=get_steps),+ get_task_actions_hydrated=AsyncMock(side_effect=get_actions),+ get_run=AsyncMock(return_value=SimpleNamespace(task_run_type=RunType.task_v1)),+ )+ monkeypatch.setattr(+ shadow.app,+ "DATABASE",+ SimpleNamespace(+ tasks=tasks_repo,+ artifacts=SimpleNamespace(get_artifacts_for_task_step=AsyncMock(return_value=[])),+ observer=SimpleNamespace(get_workflow_run_block_by_task_id=AsyncMock(return_value=None)),+ workflow_runs=SimpleNamespace(get_workflow_run_output_parameters=AsyncMock(return_value=[])),+ ),+ )+ monkeypatch.setattr(shadow.app, "ARTIFACT_MANAGER", SimpleNamespace(retrieve_artifact=AsyncMock()))+ monkeypatch.setattr(shadow.settings, "BROWSER_TYPE", "chromium-headless")++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=_har(offset_seconds=12),+ browser_state=_browser_state(),+ last_step=latest_step,+ workflow_run=_workflow_run(),+ )++ event = next(record for record in logs if record["event"] == "submission_signal_shadow")+ assert event["signal"] == "submitted_verified"+ assert event["task_id"] == "task-a"+ tasks_repo.get_run.assert_awaited_once_with(run_id="task-b", organization_id="org-1")+++async def test_runner_swallows_errors_without_logging_exception_messages(monkeypatch: pytest.MonkeyPatch) -> None:+ secret = "raw-value?token=do-not-log"+ database = SimpleNamespace(+ tasks=SimpleNamespace(get_task_steps=AsyncMock(side_effect=ValueError(secret))),+ artifacts=SimpleNamespace(),+ observer=SimpleNamespace(),+ )+ monkeypatch.setattr(shadow.app, "DATABASE", database)++ with capture_logs() as logs:+ await shadow.run_submission_signal_shadow(+ har_data=b"",+ browser_state=_browser_state(),+ last_step=_step("step-1", 1),+ task=_task(),+ )++ events = [record for record in logs if record["event"] == "submission_signal_shadow"]+ assert len(events) == 1+ assert events[0]["status"] == "error"+ assert events[0]["error_stage"] == "load_tasks"+ assert events[0]["error_type"] == "ValueError"+ assert secret not in str(events[0])+ assert "error" not in events[0] or events[0]["error"] != secretdiff --git a/tests/unit/submission/test_verifier.py b/tests/unit/submission/test_verifier.pynew file mode 100644index 000000000..ce9d960bf--- /dev/null+++ b/tests/unit/submission/test_verifier.py@@ -0,0 +1,779 @@+from __future__ import annotations++import hashlib+import json+from datetime import datetime, timedelta++import pytest++from skyvern.forge.sdk.models import Step, StepStatus+from skyvern.forge.sdk.submission.models import (+ BrowserPath,+ CandidateWindow,+ DownloadEvidence,+ NetworkEvidence,+ PageConfirmationEvidence,+ SubmissionSignal,+ TierAEvaluation,+ TierBEvaluation,+ UrlTransitionEvidence,+)+from skyvern.forge.sdk.submission.verifier import (+ CandidateEvaluation,+ build_candidate_windows,+ classify_browser_path,+ combine,+ detect_submit_candidates,+ evaluate_tier_a,+ evaluate_tier_b,+ find_latest_candidate_step_pair,+)+from skyvern.schemas.steps import AgentStepOutput, BrowserMetadata+from skyvern.webeye.actions.actions import ClickAction+from skyvern.webeye.actions.responses import ActionResult++BASE_TIME = datetime(2026, 7, 9, 12, 0, 0)+++def _step(step_id: str, order: int, *, url: str | None = None) -> Step:+ created_at = BASE_TIME + timedelta(seconds=order * 20)+ return Step(+ created_at=created_at,+ modified_at=created_at + timedelta(seconds=5),+ task_id="task_1",+ step_id=step_id,+ status=StepStatus.completed,+ output=AgentStepOutput(browser_metadata=BrowserMetadata(website_url=url)),+ order=order,+ is_last=False,+ organization_id="org_1",+ )+++def _submit_click(*, step_id: str = "step_1", text: str = "Submit") -> ClickAction:+ return ClickAction(+ element_id="button_1",+ reasoning="submit",+ action_id="action_1",+ task_id="task_1",+ step_id=step_id,+ skyvern_element_data={"tagName": "button", "attributes": {}, "text": text},+ )+++def _tier_a(+ *,+ evidence: list[NetworkEvidence] | None = None,+ har_present: bool = True,+ har_parsed: bool = True,+ har_entry_count: int = 1,+ ambiguous_entry_count: int = 0,+) -> TierAEvaluation:+ return TierAEvaluation(+ evidence=evidence or [],+ har_present=har_present,+ har_parsed=har_parsed,+ har_entry_count=har_entry_count,+ ambiguous_entry_count=ambiguous_entry_count,+ )+++def _tier_b(+ evidence: list[PageConfirmationEvidence | UrlTransitionEvidence | DownloadEvidence] | None = None,+ *,+ page_confirmation_evaluated: bool = True,+) -> TierBEvaluation:+ return TierBEvaluation(+ evidence=evidence or [],+ page_confirmation_evaluated=page_confirmation_evaluated,+ )+++def _network_evidence() -> NetworkEvidence:+ return NetworkEvidence(+ origin="https://example.com",+ method="POST",+ status=201,+ started_at=BASE_TIME,+ correlated_step_id="step_1",+ )+++def _page_evidence() -> PageConfirmationEvidence:+ return PageConfirmationEvidence(+ phrase="confirmation number",+ value_sha256=hashlib.sha256(b"abc-123").hexdigest(),+ absent_pre_submit=True,+ )+++def _har_for_candidate(*, method: str = "POST", status: int = 201) -> bytes:+ return json.dumps(+ {+ "log": {+ "entries": [+ {+ "startedDateTime": "2026-07-09T12:00:22Z",+ "request": {"method": method, "url": "https://example.com/api/submit"},+ "response": {"status": status, "content": {"mimeType": "application/json"}},+ "_resourceType": "xhr",+ }+ ]+ }+ }+ ).encode()+++def _har_at(*offsets: int) -> bytes:+ return json.dumps(+ {+ "log": {+ "entries": [+ {+ "startedDateTime": (BASE_TIME + timedelta(seconds=offset)).isoformat() + "Z",+ "request": {"method": "POST", "url": "https://example.com/api/submit"},+ "response": {"status": 201, "content": {"mimeType": "application/json"}},+ "_resourceType": "xhr",+ }+ for offset in offsets+ ]+ }+ }+ ).encode()+++def _window(step_id: str, started_at: int, ended_at: int) -> CandidateWindow:+ return CandidateWindow(+ step_id=step_id,+ started_at=BASE_TIME + timedelta(seconds=started_at),+ ended_at=BASE_TIME + timedelta(seconds=ended_at),+ )+++def _combine(+ *,+ tier_a: TierAEvaluation | None = None,+ tier_b: TierBEvaluation | None = None,+ submit_intent_detected: bool = True,+ browser_path: BrowserPath = BrowserPath.SKYVERN_CREATED,+ cua_run: bool = False,+ coordinate_click: bool = False,+):+ tier_a = tier_a or _tier_a()+ return combine(+ tier_a=tier_a,+ candidate_evaluations=[+ CandidateEvaluation(+ step_id="step_1",+ tier_a=tier_a.evidence,+ tier_b=tier_b or _tier_b(),+ is_latest=True,+ )+ ],+ detected_candidate_step_ids=["step_1"],+ submit_intent_detected=submit_intent_detected,+ browser_path=browser_path,+ cua_run=cua_run,+ coordinate_click=coordinate_click,+ )+++def test_classify_browser_path_uses_first_matching_signal() -> None:+ assert (+ classify_browser_path(+ browser_session_id="session_1",+ task_browser_session_id=None,+ remote_browser_session_id="remote_1",+ task_browser_address="ws://example.test",+ needs_cdp_frame_publisher=True,+ browser_type="chromium-headless",+ )+ == BrowserPath.SESSION_ATTACHED+ )+ assert (+ classify_browser_path(+ browser_session_id=None,+ task_browser_session_id=None,+ remote_browser_session_id="remote_1",+ task_browser_address="ws://example.test",+ needs_cdp_frame_publisher=True,+ browser_type="chromium-headless",+ )+ == BrowserPath.VENDOR_REUSED+ )+ assert (+ classify_browser_path(+ browser_session_id=None,+ task_browser_session_id=None,+ remote_browser_session_id=None,+ task_browser_address=None,+ needs_cdp_frame_publisher=True,+ browser_type="chromium-headless",+ )+ == BrowserPath.CDP_CONNECT+ )+ assert (+ classify_browser_path(+ browser_session_id=None,+ task_browser_session_id=None,+ remote_browser_session_id=None,+ task_browser_address=None,+ needs_cdp_frame_publisher=False,+ browser_type="chromium-headful",+ )+ == BrowserPath.SKYVERN_CREATED+ )+ assert (+ classify_browser_path(+ browser_session_id=None,+ task_browser_session_id=None,+ remote_browser_session_id=None,+ task_browser_address=None,+ needs_cdp_frame_publisher=False,+ browser_type="other",+ )+ == BrowserPath.UNKNOWN+ )+++def test_detect_submit_candidates_and_coordinate_click_override() -> None:+ typed_submit = ClickAction(+ element_id="button_1",+ reasoning="submit",+ step_id="step_1",+ skyvern_element_data={"tagName": "input", "attributes": {"type": "submit"}, "text": "Continue"},+ )+ verb_submit = _submit_click(text="Apply now")+ false_boundary = _submit_click(text="Resubmit")+ split_phrase = ClickAction(+ element_id="button_2",+ reasoning="continue",+ step_id="step_1",+ skyvern_element_data={+ "tagName": "button",+ "attributes": {"aria-label": "order"},+ "text": "Place",+ },+ )+ coordinate_click = ClickAction(+ element_id="button_3",+ reasoning="coordinate",+ step_id="step_2",+ x=10,+ skyvern_element_data={"tagName": "button", "attributes": {"type": "submit"}, "text": "Submit"},+ )++ detected = detect_submit_candidates([typed_submit, verb_submit, false_boundary, split_phrase, coordinate_click])++ assert [candidate.step_id for candidate in detected.candidates] == ["step_1", "step_1"]+ assert detected.submit_intent_detected is True+ assert detected.coordinate_click is True+ assert "Apply now" not in detected.model_dump_json()+++def test_combine_uses_the_strongest_coherent_candidate() -> None:+ older_network = _network_evidence()+ latest_page = _page_evidence()+ latest_download = DownloadEvidence(file_count=1)+ tier_a = _tier_a(evidence=[older_network])++ verdict = combine(+ tier_a=tier_a,+ candidate_evaluations=[+ CandidateEvaluation(+ step_id="step_1",+ tier_a=[older_network],+ tier_b=_tier_b(),+ ),+ CandidateEvaluation(+ step_id="step_3",+ tier_a=[],+ tier_b=_tier_b([latest_page]),+ is_latest=True,+ ),+ ],+ detected_candidate_step_ids=["step_1", "step_3"],+ submit_intent_detected=True,+ browser_path=BrowserPath.SKYVERN_CREATED,+ )++ assert verdict.signal == SubmissionSignal.SUBMITTED_LIKELY+ assert verdict.tier_a == []+ assert verdict.tier_b == [latest_page]+ assert verdict.winning_step_id == "step_3"+ assert not any(note.startswith("winning_step_id:") for note in verdict.notes)++ verdict = combine(+ tier_a=tier_a,+ candidate_evaluations=[+ CandidateEvaluation(+ step_id="step_1",+ tier_a=[older_network],+ tier_b=_tier_b([latest_page]),+ ),+ CandidateEvaluation(+ step_id="step_3",+ tier_a=[],+ tier_b=_tier_b([latest_download]),+ is_latest=True,+ ),+ ],+ detected_candidate_step_ids=["step_1", "step_3"],+ submit_intent_detected=True,+ browser_path=BrowserPath.SKYVERN_CREATED,+ )++ assert verdict.signal == SubmissionSignal.SUBMITTED_VERIFIED+ assert verdict.tier_a == [older_network]+ assert verdict.tier_b == [latest_page]+ assert verdict.winning_step_id == "step_1"+ assert not any(note.startswith("winning_step_id:") for note in verdict.notes)+++def test_tier_a_discards_an_entry_matching_multiple_candidate_windows() -> None:+ windows = [_window("step_1", 10, 25), _window("step_3", 20, 35)]+ tier_a = evaluate_tier_a(_har_at(22), windows)++ verdict = combine(+ tier_a=tier_a,+ candidate_evaluations=[+ CandidateEvaluation(+ step_id="step_1",+ tier_a=[item for item in tier_a.evidence if item.correlated_step_id == "step_1"],+ tier_b=_tier_b([_page_evidence()]),+ ),+ CandidateEvaluation(+ step_id="step_3",+ tier_a=[item for item in tier_a.evidence if item.correlated_step_id == "step_3"],+ tier_b=_tier_b(),+ is_latest=True,+ ),+ ],+ detected_candidate_step_ids=["step_1", "step_3"],+ submit_intent_detected=True,+ browser_path=BrowserPath.SKYVERN_CREATED,+ )++ assert verdict.signal == SubmissionSignal.SUBMITTED_LIKELY+ assert tier_a.evidence == []+ assert verdict.tier_a == []+ assert verdict.winning_step_id == "step_1"+ assert "ambiguous_entries:1" in verdict.notes+++def test_tier_a_attributes_an_entry_matching_exactly_one_of_overlapping_windows() -> None:+ windows = [_window("step_1", 10, 25), _window("step_3", 20, 35)]++ evaluation = evaluate_tier_a(_har_at(12), windows)++ assert [item.correlated_step_id for item in evaluation.evidence] == ["step_1"]+ assert evaluation.ambiguous_entry_count == 0+++def test_tier_a_attributes_two_unambiguous_entries_to_different_candidates() -> None:+ windows = [_window("step_1", 10, 25), _window("step_3", 20, 35)]++ evaluation = evaluate_tier_a(_har_at(12, 32), windows)++ assert [item.correlated_step_id for item in evaluation.evidence] == ["step_1", "step_3"]+ assert evaluation.ambiguous_entry_count == 0+++def test_candidate_pair_uses_next_chronological_step_as_post_action() -> None:+ steps = [_step("step_3", 3), _step("step_1", 1), _step("step_2", 2)]+ detected = detect_submit_candidates([_submit_click(step_id="step_2")])++ windows = build_candidate_windows(detected, steps)+ pre_step, post_step = find_latest_candidate_step_pair(detected, steps)++ assert len(windows) == 1+ assert windows[0].step_id == "step_2"+ assert windows[0].started_at < steps[2].created_at+ assert pre_step is not None and pre_step.step_id == "step_2"+ assert post_step is not None and post_step.step_id == "step_3"+++def test_latest_candidate_without_a_post_step_does_not_reuse_an_older_pair() -> None:+ steps = [_step("step_1", 1), _step("step_2", 2), _step("step_3", 3)]+ detected = detect_submit_candidates([_submit_click(step_id="step_1"), _submit_click(step_id="step_3")])++ pre_step, post_step = find_latest_candidate_step_pair(detected, steps)++ assert pre_step is not None and pre_step.step_id == "step_3"+ assert post_step is None+++def test_tier_a_filters_non_submission_entries_and_normalizes_timestamps() -> None:+ step = _step("step_1", 1)+ windows = build_candidate_windows(detect_submit_candidates([_submit_click()]), [step])++ def entry(+ url: str,+ *,+ method: str = "POST",+ status: int = 200,+ resource_type: str | None = "xhr",+ started_at: str = "2026-07-09T12:00:22Z",+ mime_type: str = "application/json",+ ) -> dict[str, object]:+ value: dict[str, object] = {+ "startedDateTime": started_at,+ "request": {"method": method, "url": url},+ "response": {"status": status, "content": {"mimeType": mime_type}},+ }+ if resource_type is not None:+ value["_resourceType"] = resource_type+ return value++ har_bytes = json.dumps(+ {+ "log": {+ "entries": [+ entry("https://example.com/api/submit?token=secret", status=201),+ entry("https://example.com/api/read", method="GET"),+ entry("https://example.com/api/rejected", status=404),+ entry("https://example.com/api/fail", status=500),+ entry("https://example.com/image.png", resource_type="image", mime_type="image/png"),+ entry("https://example.com/styles.css", resource_type=None, mime_type="text/css"),+ entry(+ "https://example.com/scripts/app.js",+ resource_type=None,+ mime_type="application/javascript",+ ),+ entry("https://example.com/api/late", started_at="2026-07-09T12:05:00+00:00"),+ ]+ }+ }+ ).encode()++ evaluation = evaluate_tier_a(har_bytes, windows)++ assert evaluation.har_present is True+ assert evaluation.har_parsed is True+ assert evaluation.har_entry_count == 8+ assert len(evaluation.evidence) == 1+ assert evaluation.evidence[0].origin == "https://example.com"+ assert evaluation.evidence[0].correlated_step_id == "step_1"++[email protected](("method", "status"), [("POST", 201), ("PUT", 204), ("PATCH", 302)])+def test_tier_a_accepts_each_submission_method_and_success_status(method: str, status: int) -> None:+ step = _step("step_1", 1)+ windows = build_candidate_windows(detect_submit_candidates([_submit_click()]), [step])+ har_bytes = json.dumps(+ {+ "log": {+ "entries": [+ {+ "startedDateTime": "2026-07-09T12:00:22Z",+ "request": {"method": method, "url": "https://example.com/api/submit"},+ "response": {"status": status, "content": {"mimeType": "application/json"}},+ "_resourceType": "fetch",+ }+ ]+ }+ }+ ).encode()++ evaluation = evaluate_tier_a(har_bytes, windows)++ assert [item.method for item in evaluation.evidence] == [method]+++def test_tier_a_malformed_har_is_unavailable_not_an_exception() -> None:+ evaluation = evaluate_tier_a(b"not-json", [])++ assert evaluation.har_present is True+ assert evaluation.har_parsed is False+ assert evaluation.har_entry_count == 0+ assert evaluation.evidence == []+++def test_tier_a_excessively_nested_har_is_unavailable_not_an_exception() -> None:+ deeply_nested_json = b'{"x":' * 10_000 + b"0" + b"}" * 10_000++ evaluation = evaluate_tier_a(deeply_nested_json, [])++ assert evaluation.har_present is True+ assert evaluation.har_parsed is False+ assert evaluation.har_entry_count == 0+++def test_tier_a_empty_har_is_missing() -> None:+ evaluation = evaluate_tier_a(b"", [])++ assert evaluation.har_present is False+ assert evaluation.har_parsed is False+ assert evaluation.har_entry_count == 0+++def test_tier_b_detects_independent_evidence_without_retaining_raw_values() -> None:+ fake_value = "abc-123"+ evaluation = evaluate_tier_b(+ pre_url="https://example.test/apply/[email protected]/token-before?draft=private#form",+ post_url="https://example.test/complete/[email protected]/token-after?token=private#receipt",+ action_results=[ActionResult(success=True, download_triggered=True, downloaded_files=["private-receipt.pdf"])],+ pre_page_text="Application ready",+ post_page_text=f"Thank you. Confirmation number: {fake_value}",+ )++ assert evaluation.page_confirmation_evaluated is True+ assert len(evaluation.evidence) == 3+ url_evidence = next(evidence for evidence in evaluation.evidence if isinstance(evidence, UrlTransitionEvidence))+ confirmation = next(evidence for evidence in evaluation.evidence if isinstance(evidence, PageConfirmationEvidence))+ download = next(evidence for evidence in evaluation.evidence if isinstance(evidence, DownloadEvidence))+ assert url_evidence.from_origin == "https://example.test"+ assert url_evidence.to_origin == "https://example.test"+ assert url_evidence.path_changed is True+ assert confirmation.value_sha256 == hashlib.sha256(fake_value.encode()).hexdigest()+ assert download.file_count == 1+ assert fake_value not in evaluation.model_dump_json()+ assert "private-receipt.pdf" not in evaluation.model_dump_json()+ assert "token=" not in evaluation.model_dump_json()+ assert "[email protected]" not in evaluation.model_dump_json()+ assert "token-before" not in evaluation.model_dump_json()+ assert "token-after" not in evaluation.model_dump_json()+++def test_tier_b_ignores_query_fragment_only_url_changes_and_existing_confirmation() -> None:+ evaluation = evaluate_tier_b(+ pre_url="https://example.test/form?draft=one#start",+ post_url="https://example.test/form?draft=two#end",+ action_results=[],+ pre_page_text="Thank you for reviewing your request.",+ post_page_text="Thank you for reviewing your request.",+ )++ assert evaluation.page_confirmation_evaluated is True+ assert evaluation.evidence == []+++ "result",+ [+ ActionResult(success=True, download_triggered=True),+ ActionResult(success=True, downloaded_files=["receipt.pdf"]),+ ],+)+def test_tier_b_accepts_each_download_signal(result: ActionResult) -> None:+ evaluation = evaluate_tier_b(+ pre_url=None,+ post_url=None,+ action_results=[result],+ pre_page_text="Ready",+ post_page_text="Still ready",+ )++ assert evaluation.evidence == [DownloadEvidence(file_count=1)]+++def test_all_submission_signal_values_are_reachable() -> None:+ assert (+ _combine(tier_a=_tier_a(evidence=[_network_evidence()]), tier_b=_tier_b([_page_evidence()])).signal+ == SubmissionSignal.SUBMITTED_VERIFIED+ )+ assert _combine(tier_a=_tier_a(evidence=[_network_evidence()])).signal == SubmissionSignal.SUBMITTED_LIKELY+ assert _combine(tier_b=_tier_b([_page_evidence()])).signal == SubmissionSignal.SUBMITTED_LIKELY+ assert _combine().signal == SubmissionSignal.NOT_SUBMITTED+ assert _combine(tier_a=_tier_a(har_present=False, har_parsed=False, har_entry_count=0)).signal == (+ SubmissionSignal.NOT_EVALUATED+ )+++ ("har_bytes", "post_page_text", "expected_signal"),+ [+ (_har_for_candidate(), "Thank you. Your request was received.", SubmissionSignal.SUBMITTED_VERIFIED),+ (_har_for_candidate(), "The form remains ready.", SubmissionSignal.SUBMITTED_LIKELY),+ (_har_for_candidate(method="GET"), "The form remains ready.", SubmissionSignal.NOT_SUBMITTED),+ (b"", "The form remains ready.", SubmissionSignal.NOT_EVALUATED),+ ],+)+def test_every_signal_is_reachable_from_raw_inputs(+ har_bytes: bytes,+ post_page_text: str,+ expected_signal: SubmissionSignal,+) -> None:+ step = _step("step_1", 1)+ detection = detect_submit_candidates([_submit_click()])+ tier_a = evaluate_tier_a(har_bytes, build_candidate_windows(detection, [step]))+ tier_b = evaluate_tier_b(+ pre_url="https://example.com/form",+ post_url="https://example.com/form",+ action_results=[],+ pre_page_text="Review your request.",+ post_page_text=post_page_text,+ )++ verdict = combine(+ tier_a=tier_a,+ candidate_evaluations=[+ CandidateEvaluation(+ step_id="step_1",+ tier_a=tier_a.evidence,+ tier_b=tier_b,+ is_latest=True,+ )+ ],+ detected_candidate_step_ids=["step_1"],+ submit_intent_detected=detection.submit_intent_detected,+ browser_path=BrowserPath.SKYVERN_CREATED,+ )++ assert verdict.signal == expected_signal+++def test_not_submitted_unreachable_with_ambiguous_entries() -> None:+ verdict = _combine(tier_a=_tier_a(ambiguous_entry_count=1))++ assert verdict.signal == SubmissionSignal.NOT_EVALUATED+ assert "ambiguous_entries:1" in verdict.notes+++def test_not_submitted_unreachable_when_any_candidate_page_confirmation_unknown() -> None:+ older = CandidateEvaluation(step_id="step_1", tier_a=[], tier_b=_tier_b(page_confirmation_evaluated=False))+ latest = CandidateEvaluation(step_id="step_2", tier_a=[], tier_b=_tier_b(), is_latest=True)++ verdict = combine(+ tier_a=_tier_a(),+ candidate_evaluations=[older, latest],+ detected_candidate_step_ids=["step_1", "step_2"],+ submit_intent_detected=True,+ browser_path=BrowserPath.SKYVERN_CREATED,+ )++ assert verdict.signal == SubmissionSignal.NOT_EVALUATED+++def test_not_submitted_unreachable_when_a_detected_candidate_step_is_unmapped() -> None:+ mapped = CandidateEvaluation(step_id="step_valid", tier_a=[], tier_b=_tier_b(), is_latest=True)++ verdict = combine(+ tier_a=_tier_a(),+ candidate_evaluations=[mapped],+ detected_candidate_step_ids=["step_valid", "step_missing"],+ submit_intent_detected=True,+ browser_path=BrowserPath.SKYVERN_CREATED,+ )++ assert verdict.signal == SubmissionSignal.NOT_EVALUATED+++def test_not_submitted_unreachable_when_har_is_missing() -> None:+ assert _combine(tier_a=_tier_a(har_present=False)).signal == SubmissionSignal.NOT_EVALUATED+++def test_not_submitted_unreachable_when_har_is_unparseable() -> None:+ assert _combine(tier_a=_tier_a(har_parsed=False)).signal == SubmissionSignal.NOT_EVALUATED+++def test_not_submitted_unreachable_when_har_has_zero_entries() -> None:+ assert _combine(tier_a=_tier_a(har_entry_count=0)).signal == SubmissionSignal.NOT_EVALUATED+++def test_not_submitted_unreachable_without_submit_intent() -> None:+ assert _combine(submit_intent_detected=False).signal == SubmissionSignal.NOT_EVALUATED+++def test_not_submitted_unreachable_outside_skyvern_created_path() -> None:+ assert _combine(browser_path=BrowserPath.CDP_CONNECT).signal == SubmissionSignal.NOT_EVALUATED+++def test_not_submitted_unreachable_when_page_confirmation_is_unavailable() -> None:+ assert _combine(tier_b=_tier_b(page_confirmation_evaluated=False)).signal == SubmissionSignal.NOT_EVALUATED++[email protected]("override", [{"cua_run": True}, {"coordinate_click": True}])+def test_cua_and_coordinate_clicks_are_not_evaluated(override: dict[str, bool]) -> None:+ verdict = _combine(+ tier_a=_tier_a(evidence=[_network_evidence()]),+ tier_b=_tier_b([_page_evidence()]),+ **override,+ )++ assert verdict.signal == SubmissionSignal.NOT_EVALUATED+++ "browser_path",+ [BrowserPath.CDP_CONNECT, BrowserPath.VENDOR_REUSED, BrowserPath.SESSION_ATTACHED, BrowserPath.UNKNOWN],+)+def test_non_skyvern_paths_are_tier_b_only_and_capped(browser_path: BrowserPath) -> None:+ verdict = _combine(+ tier_a=_tier_a(evidence=[_network_evidence()]),+ tier_b=_tier_b([_page_evidence(), DownloadEvidence(file_count=1)]),+ browser_path=browser_path,+ )++ assert verdict.signal == SubmissionSignal.SUBMITTED_LIKELY+ assert verdict.tier_a == []+ assert verdict.capped is True++ without_tier_b = _combine(+ tier_a=_tier_a(evidence=[_network_evidence()]),+ browser_path=browser_path,+ )+ assert without_tier_b.signal == SubmissionSignal.NOT_EVALUATED+ assert without_tier_b.tier_a == []+ assert without_tier_b.capped is True+++def test_evidence_models_only_serialize_derived_facts() -> None:+ fake_value = "fake-confirmation-value"+ fake_filename = "private-receipt.pdf"+ har_bytes = json.dumps(+ {+ "log": {+ "entries": [+ {+ "startedDateTime": "2026-07-09T12:00:22Z",+ "request": {+ "method": "POST",+ "url": "https://example.com/api/submit?token=private",+ "postData": {"text": fake_value},+ },+ "response": {+ "status": 201,+ "content": {"mimeType": "application/json", "text": fake_value},+ },+ "_resourceType": "fetch",+ }+ ]+ }+ }+ ).encode()+ step = _step("step_1", 1)+ detection = detect_submit_candidates([_submit_click()])+ tier_a = evaluate_tier_a(har_bytes, build_candidate_windows(detection, [step]))+ tier_b = evaluate_tier_b(+ pre_url="https://example.com/form/[email protected]/private-token?draft=private#start",+ post_url="https://example.com/complete/[email protected]/private-token?token=private#receipt",+ action_results=[ActionResult(success=True, downloaded_files=[fake_filename])],+ pre_page_text="Review your request.",+ post_page_text=f"Confirmation number: {fake_value}",+ )+ verdict = combine(+ tier_a=tier_a,+ candidate_evaluations=[+ CandidateEvaluation(+ step_id="step_1",+ tier_a=tier_a.evidence,+ tier_b=tier_b,+ is_latest=True,+ )+ ],+ detected_candidate_step_ids=["step_1"],+ submit_intent_detected=detection.submit_intent_detected,+ browser_path=BrowserPath.SKYVERN_CREATED,+ )++ serialized = verdict.model_dump_json()+ assert verdict.signal == SubmissionSignal.SUBMITTED_VERIFIED+ assert fake_value not in serialized+ assert fake_filename not in serialized+ assert "?" not in serialized+ assert "body" not in serialized+ assert "postData" not in serialized+ assert "[email protected]" not in serialized+ assert "private-token" not in serialized+ assert hashlib.sha256(fake_value.encode()).hexdigest() in serializeddiff --git a/tests/unit/submission/test_wiring.py b/tests/unit/submission/test_wiring.pynew file mode 100644index 000000000..35e97b69d--- /dev/null+++ b/tests/unit/submission/test_wiring.py@@ -0,0 +1,313 @@+from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.config import Settings+from skyvern.forge.agent import ForgeAgent+from skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.workflow.service import WorkflowService+from skyvern.webeye.browser_artifacts import BrowserArtifacts+++def _task(task_id: str = "tsk_1", organization_id: str = "o_1") -> MagicMock:+ task = MagicMock()+ task.task_id = task_id+ task.organization_id = organization_id+ task.webhook_callback_url = None+ return task+++def _step(step_id: str = "stp_1", task_id: str = "tsk_1") -> MagicMock:+ step = MagicMock()+ step.step_id = step_id+ step.task_id = task_id+ return step+++def _browser_state() -> MagicMock:+ browser_state = MagicMock()+ browser_state.browser_artifacts = BrowserArtifacts()+ browser_state.browser_context = None+ return browser_state+++def _workflow() -> MagicMock:+ workflow = MagicMock()+ workflow.workflow_id = "wf_1"+ return workflow+++def _workflow_run() -> MagicMock:+ workflow_run = MagicMock()+ workflow_run.workflow_run_id = "wr_1"+ return workflow_run+++def test_submission_signal_shadow_setting_defaults_off() -> None:+ assert Settings.model_fields["SKYVERN_SUBMISSION_SIGNAL_SHADOW"].default is False+++async def test_task_cleanup_does_not_schedule_submission_shadow_when_flag_is_off() -> None:+ agent = ForgeAgent()+ task = _task()+ last_step = _step()+ browser_state = _browser_state()+ scheduled: list[dict[str, object]] = []++ def schedule_submission_signal_shadow(**kwargs: object) -> None:+ scheduled.append(kwargs)++ with (+ patch("skyvern.forge.agent.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", False),+ patch(+ "skyvern.forge.agent.submission_shadow.schedule_submission_signal_shadow",+ side_effect=schedule_submission_signal_shadow,+ ),+ patch("skyvern.forge.agent.skyvern_context.current", return_value=None),+ patch("skyvern.forge.agent.app") as mock_app,+ ):+ mock_app.BROWSER_MANAGER.cleanup_for_task = AsyncMock(return_value=browser_state)+ mock_app.BROWSER_MANAGER.get_video_artifacts = AsyncMock(return_value=[])+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(return_value=b'{"log":{"entries":[]}}')+ mock_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")+ mock_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()++ await agent.cleanup_browser_and_create_artifacts(+ close_browser_on_completion=True,+ last_step=last_step,+ task=task,+ browser_session_id="pbs_1",+ )++ assert scheduled == []+++async def test_task_cleanup_schedules_submission_shadow_with_the_har_context() -> None:+ agent = ForgeAgent()+ task = _task()+ last_step = _step()+ browser_state = _browser_state()+ har_data = b'{"log":{"entries":[]}}'+ calls: list[str] = []+ scheduled: list[dict[str, object]] = []++ async def get_har_data(**_: object) -> bytes:+ calls.append("har")+ return har_data++ async def get_browser_console_log(**_: object) -> bytes:+ calls.append("browser_log")+ return b""++ def schedule_submission_signal_shadow(**kwargs: object) -> None:+ calls.append("shadow")+ scheduled.append(kwargs)++ with (+ patch("skyvern.forge.agent.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", True),+ patch(+ "skyvern.forge.agent.submission_shadow.schedule_submission_signal_shadow",+ side_effect=schedule_submission_signal_shadow,+ ),+ patch("skyvern.forge.agent.skyvern_context.current", return_value=None),+ patch("skyvern.forge.agent.app") as mock_app,+ ):+ mock_app.BROWSER_MANAGER.cleanup_for_task = AsyncMock(return_value=browser_state)+ mock_app.BROWSER_MANAGER.get_video_artifacts = AsyncMock(return_value=[])+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(side_effect=get_har_data)+ mock_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(side_effect=get_browser_console_log)+ mock_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()++ await agent.cleanup_browser_and_create_artifacts(+ close_browser_on_completion=True,+ last_step=last_step,+ task=task,+ browser_session_id="pbs_1",+ )++ assert calls[:3] == ["har", "shadow", "browser_log"]+ assert scheduled == [+ {+ "har_data": har_data,+ "browser_state": browser_state,+ "last_step": last_step,+ "task": task,+ "browser_session_id": "pbs_1",+ }+ ]+++async def test_workflow_har_cleanup_does_not_schedule_submission_shadow_when_flag_is_off() -> None:+ service = WorkflowService()+ browser_state = _browser_state()+ last_step = _step()+ workflow = _workflow()+ workflow_run = _workflow_run()+ scheduled: list[dict[str, object]] = []++ def schedule_submission_signal_shadow(**kwargs: object) -> None:+ scheduled.append(kwargs)++ with (+ patch("skyvern.forge.sdk.workflow.service.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", False),+ patch(+ "skyvern.forge.sdk.workflow.service.submission_shadow.schedule_submission_signal_shadow",+ side_effect=schedule_submission_signal_shadow,+ ),+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,+ ):+ mock_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(return_value=b'{"log":{"entries":[]}}')+ mock_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()+ mock_app.ARTIFACT_MANAGER.create_task_archive = AsyncMock()++ await service.persist_har_data(browser_state, last_step, workflow, workflow_run)+ await service._persist_debug_artifacts_bundled(browser_state, last_step, workflow, workflow_run)++ assert scheduled == []+++async def test_workflow_standalone_har_schedules_submission_shadow_with_the_har_context() -> None:+ service = WorkflowService()+ browser_state = _browser_state()+ last_step = _step()+ workflow = _workflow()+ workflow_run = _workflow_run()+ har_data = b'{"log":{"entries":[]}}'+ calls: list[str] = []+ scheduled: list[dict[str, object]] = []++ async def get_har_data(**_: object) -> bytes:+ calls.append("har")+ return har_data++ def schedule_submission_signal_shadow(**kwargs: object) -> None:+ calls.append("shadow")+ scheduled.append(kwargs)++ async def create_artifact(**_: object) -> None:+ calls.append("artifact")++ with (+ patch("skyvern.forge.sdk.workflow.service.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", True),+ patch(+ "skyvern.forge.sdk.workflow.service.submission_shadow.schedule_submission_signal_shadow",+ side_effect=schedule_submission_signal_shadow,+ ),+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,+ ):+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(side_effect=get_har_data)+ mock_app.ARTIFACT_MANAGER.create_artifact = AsyncMock(side_effect=create_artifact)++ await service.persist_har_data(browser_state, last_step, workflow, workflow_run)++ assert calls == ["har", "shadow", "artifact"]+ assert scheduled == [+ {+ "har_data": har_data,+ "browser_state": browser_state,+ "last_step": last_step,+ "workflow_run": workflow_run,+ }+ ]+++async def test_workflow_bundled_har_schedules_submission_shadow_with_the_har_context() -> None:+ service = WorkflowService()+ browser_state = _browser_state()+ last_step = _step()+ workflow = _workflow()+ workflow_run = _workflow_run()+ har_data = b'{"log":{"entries":[]}}'+ calls: list[str] = []+ scheduled: list[dict[str, object]] = []++ async def get_har_data(**_: object) -> bytes:+ calls.append("har")+ return har_data++ def schedule_submission_signal_shadow(**kwargs: object) -> None:+ calls.append("shadow")+ scheduled.append(kwargs)++ async def create_task_archive(**_: object) -> None:+ calls.append("archive")++ with (+ patch("skyvern.forge.sdk.workflow.service.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", True),+ patch(+ "skyvern.forge.sdk.workflow.service.submission_shadow.schedule_submission_signal_shadow",+ side_effect=schedule_submission_signal_shadow,+ ),+ patch("skyvern.forge.sdk.workflow.service.app") as mock_app,+ ):+ mock_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")+ mock_app.BROWSER_MANAGER.get_har_data = AsyncMock(side_effect=get_har_data)+ mock_app.ARTIFACT_MANAGER.create_task_archive = AsyncMock(side_effect=create_task_archive)++ await service._persist_debug_artifacts_bundled(browser_state, last_step, workflow, workflow_run)++ assert calls == ["har", "shadow", "archive"]+ assert scheduled == [+ {+ "har_data": har_data,+ "browser_state": browser_state,+ "last_step": last_step,+ "workflow_run": workflow_run,+ }+ ]+++async def test_wiring_persists_har_when_shadow_scheduler_setup_fails() -> None:+ agent = ForgeAgent()+ service = WorkflowService()+ task = _task()+ browser_state = _browser_state()+ last_step = _step()+ workflow = _workflow()+ workflow_run = _workflow_run()+ har_data = b'{"log":{"entries":[]}}'++ with (+ patch("skyvern.forge.sdk.submission.shadow.settings.SKYVERN_SUBMISSION_SIGNAL_SHADOW", True),+ patch(+ "skyvern.forge.sdk.submission.shadow._prune_pending",+ side_effect=RuntimeError("scheduler setup failed"),+ ),+ patch("skyvern.forge.agent.skyvern_context.current", return_value=None),+ patch("skyvern.forge.agent.app") as agent_app,+ patch("skyvern.forge.sdk.workflow.service.app") as workflow_app,+ ):+ agent_app.BROWSER_MANAGER.cleanup_for_task = AsyncMock(return_value=browser_state)+ agent_app.BROWSER_MANAGER.get_video_artifacts = AsyncMock(return_value=[])+ agent_app.BROWSER_MANAGER.get_har_data = AsyncMock(return_value=har_data)+ agent_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")+ agent_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()+ workflow_app.BROWSER_MANAGER.get_browser_console_log = AsyncMock(return_value=b"")+ workflow_app.BROWSER_MANAGER.get_har_data = AsyncMock(return_value=har_data)+ workflow_app.ARTIFACT_MANAGER.create_artifact = AsyncMock()+ workflow_app.ARTIFACT_MANAGER.create_task_archive = AsyncMock()++ await agent.cleanup_browser_and_create_artifacts(True, last_step, task, browser_session_id="pbs_1")+ await service.persist_har_data(browser_state, last_step, workflow, workflow_run)+ await service._persist_debug_artifacts_bundled(browser_state, last_step, workflow, workflow_run)++ agent_app.ARTIFACT_MANAGER.create_artifact.assert_awaited_once_with(+ step=last_step, artifact_type=ArtifactType.HAR, data=har_data+ )+ workflow_app.ARTIFACT_MANAGER.create_artifact.assert_awaited_once_with(+ step=last_step, artifact_type=ArtifactType.HAR, data=har_data+ )+ workflow_app.ARTIFACT_MANAGER.create_task_archive.assert_awaited_once_with(+ step=last_step,+ entries={"har.har": (ArtifactType.HAR, har_data)},+ workflow_run_id=workflow_run.workflow_run_id,+ )