Skip to content

Automate browser based workflows with AI

Segment managed browser profiles by rotating credential (SKY-15192) (#8317)

Aaron Perez0db2c1961224
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index d1ee92749..835e9c2d7 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -3158,14 +3158,14 @@
"type": "null"
}
],
- "description": "Exact-match filter on a workflow run's top classifier failure category (`failure_category[0].category`). Only failed, terminated, and timed_out runs are eligible; canceled runs are never returned by this filter.",
+ "description": "Exact-match filter on a workflow run's top classifier failure category (`failure_category[0].category`). Only failed, terminated, and timed_out runs are eligible; canceled runs are never returned by this filter. Applies to `run_type=workflow_run` only — combined with a task run_type it matches nothing.",
"examples": [
"ANTI_BOT_DETECTION",
"AUTH_FAILURE"
],
"title": "Failure Category"
},
- "description": "Exact-match filter on a workflow run's top classifier failure category (`failure_category[0].category`). Only failed, terminated, and timed_out runs are eligible; canceled runs are never returned by this filter."
+ "description": "Exact-match filter on a workflow run's top classifier failure category (`failure_category[0].category`). Only failed, terminated, and timed_out runs are eligible; canceled runs are never returned by this filter. Applies to `run_type=workflow_run` only — combined with a task run_type it matches nothing."
},
{
"name": "x-api-key",
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 1d6a2dd16..fd5b0ec89 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -69,6 +69,8 @@ PostRunPagePathKind = Literal["login", "challenge", "incomplete_navigation", "no
PostRunPagePathTargetKind = Literal["form_submit", "navigation", "clickable", "challenge"]
BuildTestPacketWorkflowSource = Literal["accepted_write_readback", "turn_start_persisted_readback", "unavailable"]
BuildTestPacketUnfinishedKind = Literal["unverified_block", "missing_requested_output"]
+BuildTestPacketPageCaptureStatus = Literal["captured", "unavailable"]
+BuildTestPacketPageCaptureOmission = Literal["screenshot_capture_failed", "page_capture_unavailable"]
BuildTestPacketLocatorUnobservedReason = Literal[
"worker_owned_run",
"run_browser_unavailable",
@@ -128,6 +130,7 @@ class BuildTestPacketRun(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
workflow_run_id: str | None = None
+ browser_session_id: str | None = None
status: str | None = None
browser: BuildTestPacketRunBrowser | None = None
@@ -148,6 +151,26 @@ class BuildTestPacketPageState(BaseModel):
obstructions: list[PageObstruction] = Field(default_factory=list)
+class BuildTestPacketPageCapture(BaseModel):
+ """The factual result of trying to retain post-run page evidence.
+
+ This is transport metadata only. It does not classify the run or choose a repair action.
+ """
+
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ status: BuildTestPacketPageCaptureStatus
+ omission: BuildTestPacketPageCaptureOmission | None = None
+
+ @model_validator(mode="after")
+ def validate_capture_state(self) -> BuildTestPacketPageCapture:
+ if self.status == "unavailable" and self.omission != "page_capture_unavailable":
+ raise ValueError("an unavailable page capture must carry page_capture_unavailable")
+ if self.status == "captured" and self.omission == "page_capture_unavailable":
+ raise ValueError("a captured page cannot carry a page-unavailable omission")
+ return self
+
+
class BuildTestPacketLocatorObservation(BaseModel):
"""What one authored locator resolved to on the page the failure left behind.
@@ -255,6 +278,7 @@ class BuildTestEvidencePacket(BaseModel):
action_observations: list[str] = Field(default_factory=list)
failure: BuildTestPacketFailure | None = None
page_state: BuildTestPacketPageState | None = None
+ page_capture: BuildTestPacketPageCapture | None = None
requested_outputs: list[BuildTestPacketRequestedOutput] = Field(default_factory=list)
registered_outputs: list[BuildTestPacketRegisteredOutput] = Field(default_factory=list)
downloads: list[BuildTestPacketDownload] = Field(default_factory=list)
@@ -281,6 +305,7 @@ class RecordedBuildTestOutcome(BaseModel):
structural_failure_identity: str = ""
verified_progress_marker: str = ""
page_evidence_refs: list[str] = Field(default_factory=list)
+ page_capture: BuildTestPacketPageCapture | None = None
evidence_refs: list[str] = Field(default_factory=list)
missing_requested_output_facts: list[dict[str, object]] = Field(default_factory=list)
runtime_output_repair_facts: list[dict[str, object]] = Field(default_factory=list)
@@ -785,6 +810,7 @@ def _required_input_unbound_outcome(
requested_block_labels: list[str],
block_shape_hashes: Mapping[str, str],
workflow_run_id: str | None,
+ page_capture: BuildTestPacketPageCapture | None,
authored_structure_signature: str | None,
referenced_unbound_keys: Sequence[str],
) -> RecordedBuildTestOutcome:
@@ -799,6 +825,7 @@ def _required_input_unbound_outcome(
requested_block_labels=requested_block_labels,
block_shape_hashes=dict(block_shape_hashes),
structural_failure_identity=_required_input_unbound_identity(failed_block, referenced_unbound_keys),
+ page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=_bounded_text(
"unbound required inputs: " + ", ".join(_clean_list(referenced_unbound_keys))
@@ -840,16 +867,9 @@ def recorded_outcome_from_run_blocks_result(
unbound_required_parameter_keys or [],
block_parameter_keys or {},
)
- graded_page_evidence = (
- page_evidence
- if page_evidence is not None
- and page_evidence_source_matches_run(
- _safe_str(page_evidence.get("source_browser_session_id")),
- _safe_str(data.get("browser_session_id")),
- )
- else None
- )
+ graded_page_evidence = page_evidence if _post_run_page_evidence_matches_result(data, page_evidence) else None
page_refs = _page_evidence_refs(graded_page_evidence)
+ page_capture = post_run_page_capture_from_result(data, graded_page_evidence)
output_refs = _output_evidence_refs(blocks)
verification_identity = _completion_verification_identity(completion_verification)
authoritative_workflow_run_id = (
@@ -912,6 +932,7 @@ def recorded_outcome_from_run_blocks_result(
block_labels=block_labels,
requested_block_labels=requested_block_labels,
block_shape_hashes=block_shape_hashes,
+ page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "terminal blocker precedence suppresses repair prompt"},
@@ -927,6 +948,7 @@ def recorded_outcome_from_run_blocks_result(
requested_block_labels=requested_block_labels,
block_shape_hashes=block_shape_hashes,
verified_progress_marker=verification_identity or "run_completed_verified",
+ page_capture=page_capture,
evidence_refs=output_refs,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "Completion verification passed.",
@@ -945,6 +967,7 @@ def recorded_outcome_from_run_blocks_result(
block_labels=block_labels,
requested_block_labels=requested_block_labels,
block_shape_hashes=block_shape_hashes,
+ page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "run outcome was not evaluated"},
@@ -956,6 +979,7 @@ def recorded_outcome_from_run_blocks_result(
requested_block_labels,
block_shape_hashes,
recorded_run_outcome.workflow_run_id or workflow_run_id or None,
+ page_capture,
authored_structure_signature,
referenced_unbound_keys,
)
@@ -971,6 +995,7 @@ def recorded_outcome_from_run_blocks_result(
block_labels=block_labels,
requested_block_labels=requested_block_labels,
block_shape_hashes=block_shape_hashes,
+ page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "no typed verification/page/output identity available"},
@@ -989,6 +1014,7 @@ def recorded_outcome_from_run_blocks_result(
block_labels=block_labels,
requested_block_labels=requested_block_labels,
block_shape_hashes=block_shape_hashes,
+ page_capture=page_capture,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "turn-unsatisfiable fallback floor, no reachable route"},
@@ -1004,6 +1030,7 @@ def recorded_outcome_from_run_blocks_result(
block_shape_hashes=block_shape_hashes,
structural_failure_identity=structural_identity,
page_evidence_refs=page_refs,
+ page_capture=page_capture,
evidence_refs=evidence_refs,
missing_requested_output_facts=missing_output_facts,
runtime_output_repair_facts=runtime_output_facts,
@@ -1039,6 +1066,7 @@ def recorded_outcome_from_run_blocks_result(
requested_block_labels,
block_shape_hashes,
workflow_run_id or None,
+ page_capture,
authored_structure_signature,
referenced_unbound_keys,
)
@@ -1108,6 +1136,7 @@ def recorded_outcome_from_run_blocks_result(
block_shape_hashes=block_shape_hashes,
structural_failure_identity=structural_identity,
page_evidence_refs=page_refs,
+ page_capture=page_capture,
evidence_refs=output_refs,
missing_requested_output_facts=missing_output_facts,
authored_structure_signature=authored_structure_signature,
@@ -1447,6 +1476,48 @@ def _page_evidence_refs(page_evidence: Mapping[str, object] | None) -> list[str]
return refs[:12]
+def post_run_page_capture_from_result(
+ data: Mapping[str, object],
+ page_evidence: Mapping[str, object] | None,
+) -> BuildTestPacketPageCapture | None:
+ """Read only the typed post-run capture fact carried by the run result.
+
+ A usable page packet establishes that capture reached the page. Screenshot availability is a
+ separate fact, so its omission must not erase the page or action facts already retained.
+ """
+ raw_capture = data.get("post_run_page_capture")
+ if isinstance(raw_capture, Mapping):
+ try:
+ return BuildTestPacketPageCapture.model_validate(raw_capture)
+ except ValueError:
+ return None
+ if not _post_run_page_evidence_matches_result(data, page_evidence):
+ return None
+ assert page_evidence is not None
+ omissions = page_evidence.get("visual_capture_omissions")
+ if isinstance(omissions, list) and "screenshot_capture_failed" in omissions:
+ return BuildTestPacketPageCapture(status="captured", omission="screenshot_capture_failed")
+ return BuildTestPacketPageCapture(status="captured")
+
+
+def _post_run_page_evidence_matches_result(
+ data: Mapping[str, object],
+ page_evidence: Mapping[str, object] | None,
+) -> bool:
+ """Only evidence stamped after this result's run can establish that capture succeeded."""
+ workflow_run_id = _safe_str(data.get("workflow_run_id"))
+ return bool(
+ page_evidence is not None
+ and workflow_run_id
+ and page_evidence.get("observed_after_workflow_run") is True
+ and page_evidence.get("workflow_run_id") == workflow_run_id
+ and page_evidence_source_matches_run(
+ _safe_str(page_evidence.get("source_browser_session_id")),
+ _safe_str(data.get("browser_session_id")),
+ )
+ )
+
+
def _post_run_page_path_failure(
page_evidence: Mapping[str, object] | None,
workflow_run_id: str | None,
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 931440d43..fdfe996cc 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -924,6 +924,12 @@ def project_build_test_packet_for_llm(packet: BuildTestEvidencePacket) -> BuildT
max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
notices=notices,
),
+ "browser_session_id": _bounded_packet_string(
+ projected.run.browser_session_id,
+ field_name="run.browser_session_id",
+ max_chars=_BUILD_TEST_IDENTIFIER_MAX_CHARS,
+ notices=notices,
+ ),
"status": _bounded_packet_string(
projected.run.status,
field_name="run.status",
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index d5272b663..7c7d4014d 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -312,8 +312,9 @@ async def _augment_composition_evidence_with_visual_fallback(
screenshot_result = await _composition_get_screenshot(ctx, dispatch_session_id=capture_session_id)
if not screenshot_result.get("ok"):
return (
- _composition_add_evidence_omission(
+ _composition_add_visual_capture_omission(
evidence,
+ "screenshot_capture_failed",
f"screenshot_capture_failed: {screenshot_result.get('error', 'unknown')}",
),
None,
@@ -356,6 +357,22 @@ def _composition_add_evidence_omission(evidence: dict[str, Any], message: str) -
return merged
+def _composition_add_visual_capture_omission(
+ evidence: dict[str, Any],
+ code: str,
+ message: str,
+) -> dict[str, Any]:
+ """Record a bounded typed capture fact alongside its operator-facing detail."""
+ merged = _composition_add_evidence_omission(evidence, message)
+ omission_codes = [
+ item for item in merged.get("visual_capture_omissions") or [] if item in {"screenshot_capture_failed"}
+ ]
+ if code == "screenshot_capture_failed":
+ omission_codes.append(code)
+ merged["visual_capture_omissions"] = list(dict.fromkeys(omission_codes))[:1]
+ return merged
+
+
def _composition_add_inspection_warning(evidence: dict[str, Any], message: str) -> dict[str, Any]:
merged = dict(evidence)
warnings = [item for item in merged.get("inspection_warnings") or [] if isinstance(item, str)]
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 903c26b0e..1d5d815ab 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -34,6 +34,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestPacketFailure,
BuildTestPacketLocatorObservation,
BuildTestPacketLocatorUnobservedReason,
+ BuildTestPacketPageCapture,
BuildTestPacketPageState,
BuildTestPacketRegisteredOutput,
BuildTestPacketRequestedOutput,
@@ -44,6 +45,7 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
authored_block_parameter_keys_from_workflow,
authored_structure_signature_from_workflow,
+ post_run_page_capture_from_result,
record_build_test_outcome,
recorded_outcome_from_run_blocks_result,
unresolved_runtime_block_failure_with_disposition,
@@ -1786,13 +1788,13 @@ async def _capture_and_store_post_run_page(
run_session_id: str,
run_id: str,
current_url: str,
-) -> None:
+) -> BuildTestPacketPageCapture:
"""A failed or hollow capture neutralizes stale evidence to None only when it would not cleanly match
this run_id, so the matcher's destructive clear cannot fire on the pending failure-string context."""
registry = ctx.origin_run_redaction_registry
if registry is not None and registry.workflow_run_id == run_id and registry.contains_sensitive_values:
ctx.composition_page_evidence = None
- return
+ return BuildTestPacketPageCapture(status="unavailable", omission="page_capture_unavailable")
evidence, observed_session_id, _, captured_frame = await _read_run_session_page_evidence(
ctx, run_session_id=run_session_id, current_url=current_url
)
@@ -1805,9 +1807,11 @@ async def _capture_and_store_post_run_page(
source_browser_session_id=observed_session_id,
run_browser_session_id=run_session_id,
)
+ same_run_stored_evidence = _same_run_page_evidence_for_result(ctx, run_id)
if (
captured_frame is not None
and not preserved_stored_evidence
+ and same_run_stored_evidence is not None
and not (
ctx.origin_run_redaction_registry is not None
and ctx.origin_run_redaction_registry.workflow_run_id == run_id
@@ -1831,9 +1835,14 @@ async def _capture_and_store_post_run_page(
),
captured_at=captured_frame.captured_at,
)
- return
+ stored_capture = post_run_page_capture_from_result(
+ {"workflow_run_id": run_id, "browser_session_id": run_session_id},
+ same_run_stored_evidence,
+ )
+ return stored_capture or BuildTestPacketPageCapture(status="unavailable", omission="page_capture_unavailable")
if not post_run_inspection_cleanly_matches(ctx.composition_page_evidence, run_id):
ctx.composition_page_evidence = None
+ return BuildTestPacketPageCapture(status="unavailable", omission="page_capture_unavailable")
def _pre_run_baseline_is_provenance_valid(evidence: Mapping[str, Any] | None) -> bool:
@@ -3176,6 +3185,7 @@ async def _run_blocks_and_collect_debug(
LOG.debug("Failed to capture post-run screenshot", exc_info=True)
locator_observations: list[AuthoredLocatorObservationRow] | None = None
+ post_run_page_capture: BuildTestPacketPageCapture | None = None
if not sensitive_origin_run:
if (
not dispatch_to_worker
@@ -3186,7 +3196,7 @@ async def _run_blocks_and_collect_debug(
# CDP capture against the run session: worker-owned for dispatched runs, so skip it.
# A dispatched run therefore carries no locator observation, and the packet says so.
_pin_pre_run_page_reference(ctx, workflow_run.workflow_run_id)
- await _capture_and_store_post_run_page(
+ post_run_page_capture = await _capture_and_store_post_run_page(
ctx,
run_session_id=run_session_id,
run_id=workflow_run.workflow_run_id,
@@ -3245,6 +3255,8 @@ async def _run_blocks_and_collect_debug(
post_run_page_evidence = _same_run_page_evidence_for_result(ctx, workflow_run.workflow_run_id)
if post_run_page_evidence is not None:
result_data["post_run_page_evidence"] = model_visible_composition_evidence(post_run_page_evidence)
+ if post_run_page_capture is not None:
+ result_data["post_run_page_capture"] = post_run_page_capture.model_dump(mode="json")
_attach_run_session_facts(
result_data,
used_fresh_run_session=used_fresh_run_session,
@@ -4777,6 +4789,13 @@ def build_test_evidence_packet(
if not action_observations:
omission_notices.append("action_observations empty: no same-run action observation was recorded.")
page_state = _packet_page_state(data, omission_notices)
+ raw_page_evidence = data.get("post_run_page_evidence")
+ page_capture = post_run_page_capture_from_result(
+ data,
+ raw_page_evidence if isinstance(raw_page_evidence, Mapping) else None,
+ )
+ if page_capture is None and recorded_outcome is not None:
+ page_capture = recorded_outcome.page_capture
failure: BuildTestPacketFailure | None = None
if failed_block is not None or result.get("ok") is False:
failure = BuildTestPacketFailure(
@@ -4852,10 +4871,16 @@ def build_test_evidence_packet(
canonical_workflow_yaml_complete=workflow_yaml is not None,
attempted_block_labels=attempted_labels,
executed_block_labels=executed_labels,
- run=BuildTestPacketRun(workflow_run_id=run_id, status=run_status, browser=_packet_run_browser(data)),
+ run=BuildTestPacketRun(
+ workflow_run_id=run_id,
+ browser_session_id=_packet_string(data.get("browser_session_id")),
+ status=run_status,
+ browser=_packet_run_browser(data),
+ ),
action_observations=action_observations,
failure=failure,
page_state=page_state if failure is None else None,
+ page_capture=page_capture,
requested_outputs=requested_outputs,
registered_outputs=registered_outputs,
downloads=downloads,
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 113247dbc..c73191797 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -843,7 +843,11 @@ class WorkflowRunsRepository(BaseRepository):
# canceled run, but that's an implementation detail, not the contract — restrict to
# the failure statuses explicitly so a canceled run is never returned regardless.
category_match_clause = None
- failure_statuses = ("failed", "terminated", "timed_out")
+ failure_statuses = (
+ WorkflowRunStatus.failed.value,
+ WorkflowRunStatus.terminated.value,
+ WorkflowRunStatus.timed_out.value,
+ )
if failure_category:
top_category = (
func.json_extract(WorkflowRunModel.failure_category, "$[0].category")
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 865772991..9cbe3338f 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -4318,7 +4318,8 @@ async def get_runs_v2(
description=(
"Exact-match filter on a workflow run's top classifier failure category "
"(`failure_category[0].category`). Only failed, terminated, and timed_out runs are "
- "eligible; canceled runs are never returned by this filter."
+ "eligible; canceled runs are never returned by this filter. Applies to "
+ "`run_type=workflow_run` only — combined with a task run_type it matches nothing."
),
examples=["ANTI_BOT_DETECTION", "AUTH_FAILURE"],
),
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 4dbd3383c..3b556051a 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -2190,6 +2190,38 @@ class WorkflowService:
and (bool(parameter.credential_ids) or bool(parameter.fallback_credential_ids))
]
+ def _rotating_credential_profile_segment(self, workflow: Workflow, parameter_values: dict[str, Any]) -> str | None:
+ """Segment folded into a managed browser profile's key so each credential in a rotation pool
+ gets its own profile automatically, even when browser_profile_key doesn't reference it or
+ isn't set (SKY-15192). Reads the resolved selection already in parameter_values; a parameter
+ with no resolved selection (fail-open legacy pool) is skipped, not guessed. Joining multiple
+ selections with "," is unambiguous because a resolved value is always a generate_credential_id
+ row (cred_<int>), never free text that could itself contain a comma."""
+ segments = [
+ selected
+ for parameter in self._get_rotating_credential_parameters(workflow)
+ if isinstance(selected := parameter_values.get(parameter.key), str) and selected
+ ]
+ return ",".join(segments) or None
+
+ def _managed_browser_profile_digest_key(
+ self, workflow: Workflow, parameter_values: dict[str, Any], rendered_key: str | None
+ ) -> tuple[str | None, str | None]:
+ """(digest_key, credential_segment) for a managed browser profile. rendered_key is an
+ unrestricted Jinja render of workflow-parameter values (browser_profile_key.py) — it can
+ contain any character, so joining it with credential_segment by a plain delimiter would let
+ two different (key, credential) pairs collide onto the same digest (e.g. key renders
+ "acct|1" with no credential vs. key "acct" with credential "1"). Collapse rendered_key to its
+ fixed-width digest before combining, so the split point is never ambiguous. When the two are
+ already identical (the customer's key already IS the credential value), skip combining
+ entirely and keep today's exact digest rather than gratuitously reseeding."""
+ credential_segment = self._rotating_credential_profile_segment(workflow, parameter_values)
+ if not credential_segment or rendered_key == credential_segment:
+ return rendered_key or credential_segment, credential_segment
+ if not rendered_key:
+ return credential_segment, credential_segment
+ return f"{build_browser_profile_key_digest(rendered_key)}:{credential_segment}", credential_segment
+
def _get_run_credential_parameter_overrides(
self,
*,
@@ -3308,14 +3340,21 @@ class WorkflowService:
browser_profile_key=workflow.browser_profile_key,
)
rendered_key = None
- digest = build_browser_profile_key_digest(rendered_key)
+ digest_key, credential_segment = self._managed_browser_profile_digest_key(
+ workflow, parameter_values, rendered_key
+ )
+ digest = build_browser_profile_key_digest(digest_key)
profile, created = await app.DATABASE.browser_sessions.get_or_create_managed_browser_profile(
organization_id=workflow_run.organization_id,
workflow_permanent_id=workflow.workflow_permanent_id,
browser_profile_key_digest=digest,
- name=_build_managed_browser_profile_name(workflow.title, rendered_key),
+ name=_build_managed_browser_profile_name(workflow.title, rendered_key or credential_segment),
)
- if created:
+ # A credential-segmented profile has no legacy counterpart (the wpid_ archive predates
+ # per-credential rotation and was never scoped to one credential) — seeding it would risk
+ # loading a different credential's saved state onto this one, the exact bug this segment
+ # exists to prevent. Skip seeding; the profile just starts empty and accumulates normally.
+ if created and not credential_segment:
try:
await self._seed_managed_browser_profile_from_legacy_session(
workflow=workflow,
@@ -3511,23 +3550,30 @@ class WorkflowService:
if not workflow.persist_browser_session or workflow_request.browser_profile_id:
return None
+ parameter_values = self._profile_key_render_values(workflow, workflow_request)
+ if extra_parameter_values:
+ parameter_values.update(extra_parameter_values)
+
rendered_key = None
if workflow.browser_profile_key:
- parameter_values = self._profile_key_render_values(workflow, workflow_request)
- if extra_parameter_values:
- parameter_values.update(extra_parameter_values)
rendered_key = render_browser_profile_key(workflow.browser_profile_key, parameter_values)
if not rendered_key:
return None
- digest = build_browser_profile_key_digest(rendered_key)
+ digest_key, credential_segment = self._managed_browser_profile_digest_key(
+ workflow, parameter_values, rendered_key
+ )
+ digest = build_browser_profile_key_digest(digest_key)
profile, created = await app.DATABASE.browser_sessions.get_or_create_managed_browser_profile(
organization_id=organization_id,
workflow_permanent_id=workflow.workflow_permanent_id,
browser_profile_key_digest=digest,
- name=_build_managed_browser_profile_name(workflow.title, rendered_key),
+ name=_build_managed_browser_profile_name(workflow.title, rendered_key or credential_segment),
)
- if created:
+ # See _ensure_managed_browser_profile: a credential-segmented profile has no legacy
+ # counterpart, so seeding it from the un-credentialed archive would risk leaking a
+ # different credential's saved state onto it.
+ if created and not credential_segment:
try:
await self._seed_managed_browser_profile_from_legacy_session(
workflow=workflow,
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 22f2af5e6..cdae9317e 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import json
from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -35,6 +36,7 @@ from skyvern.forge.sdk.copilot.output_utils import project_build_test_packet_for
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.runtime_authoring_repair import inject_runtime_authoring_repair_context
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
+from skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidence
from skyvern.forge.sdk.copilot.tools.run_execution import (
_authored_literal_locator_selectors,
_failed_block_code,
@@ -1838,6 +1840,287 @@ def test_a_standalone_clickable_control_reaches_the_packet_page_state_and_the_ll
assert compacted.failure.page_state.action_summaries == ["Continue to statements disabled", "Section 0"]
+def test_screenshot_ablation_replays_preserve_page_facts_without_minting_a_decision() -> None:
+ """A missing frame changes only capture facts; the recorded run's failure remains a run fact."""
+
+ def rendered_bytes(value: object) -> bytes:
+ return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
+
+ def replay(*, screenshot_present: bool) -> tuple[dict[str, object], dict[str, object], str]:
+ ctx = _locator_packet_ctx()
+ page_evidence: dict[str, object] = {
+ "workflow_run_id": "wr_capture_failure",
+ "source_browser_session_id": "pbs_capture_failure",
+ "run_browser_session_id": "pbs_capture_failure",
+ "observed_after_workflow_run": True,
+ "source_tool": "inspect_page_for_composition",
+ "current_url": "https://example.test/checkout/payment",
+ "forms": [{"submit_controls": [{"text": "Place order", "disabled": False}]}],
+ "clickable_controls": [{"text": "Continue to payment", "disabled": False}],
+ **({} if screenshot_present else {"visual_capture_omissions": ["screenshot_capture_failed"]}),
+ }
+ result: dict[str, object] = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_capture_failure",
+ "browser_session_id": "pbs_capture_failure",
+ "overall_status": "failed",
+ "requested_block_labels": ["continue_to_payment"],
+ "executed_block_labels": ["continue_to_payment"],
+ "blocks": [
+ {
+ "label": "continue_to_payment",
+ "status": "failed",
+ "failure_reason": "Click did not reach payment options.",
+ }
+ ],
+ "post_run_page_evidence": page_evidence,
+ **({"screenshot_base64": "c2NyZWVuc2hvdA=="} if screenshot_present else {}),
+ },
+ }
+ stored, preserved = store_post_run_page_evidence(
+ ctx,
+ page_evidence,
+ run_id="wr_capture_failure",
+ current_url="https://example.test/checkout/payment",
+ source_browser_session_id="pbs_capture_failure",
+ run_browser_session_id="pbs_capture_failure",
+ )
+ assert preserved is False
+ assert stored["observed_after_workflow_run"] is True
+ result["data"]["post_run_page_evidence"] = run_execution_module._same_run_page_evidence_for_result(
+ ctx, "wr_capture_failure"
+ )
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ page_evidence=result["data"]["post_run_page_evidence"],
+ )
+ assert outcome is not None
+ packet = project_build_test_packet_for_llm(
+ build_test_evidence_packet(ctx, result, recorded_outcome=outcome)
+ ).model_dump(mode="json", exclude_none=True)
+ ordinary_input = _build_user_context(
+ workflow_yaml=ctx.workflow_yaml,
+ chat_history_text="",
+ global_llm_context="",
+ debug_run_info_text=_prior_run_debug_text(packet),
+ user_message="Repair the recorded run.",
+ )
+ return outcome.model_dump(mode="json"), packet, ordinary_input
+
+ with_screenshot = [replay(screenshot_present=True) for _ in range(3)]
+ without_screenshot = [replay(screenshot_present=False) for _ in range(3)]
+ assert len({rendered_bytes(replay_outcome) for replay_outcome, _, _ in with_screenshot}) == 1
+ assert len({rendered_bytes(replay_outcome) for replay_outcome, _, _ in without_screenshot}) == 1
+ assert len({rendered_bytes(packet) for _, packet, _ in with_screenshot}) == 1
+ assert len({rendered_bytes(packet) for _, packet, _ in without_screenshot}) == 1
+
+ baseline_outcome, baseline_packet, _ = with_screenshot[0]
+ missing_outcome, missing_packet, ordinary_input = without_screenshot[0]
+ assert {key: value for key, value in baseline_outcome.items() if key != "page_capture"} == {
+ key: value for key, value in missing_outcome.items() if key != "page_capture"
+ }
+ assert missing_packet["page_capture"] == {"status": "captured", "omission": "screenshot_capture_failed"}
+ assert baseline_packet["screenshot"]["present"] is True
+ assert missing_packet["screenshot"]["present"] is False
+ assert {
+ key: value
+ for key, value in baseline_packet.items()
+ if key not in {"page_capture", "screenshot", "omission_notices"}
+ } == {
+ key: value
+ for key, value in missing_packet.items()
+ if key not in {"page_capture", "screenshot", "omission_notices"}
+ }
+ assert not {"success_verdict", "terminal", "repair_decision"}.intersection(missing_packet)
+ decision_surface = {
+ key: value
+ for key, value in missing_packet.items()
+ if key not in {"page_capture", "screenshot", "omission_notices"}
+ }
+ assert b"success_verdict" not in rendered_bytes(decision_surface)
+ assert b"terminal" not in rendered_bytes(decision_surface)
+ assert b"repair_decision" not in rendered_bytes(decision_surface)
+ assert '"workflow_run_id": "wr_capture_failure"' in ordinary_input
+ assert '"browser_session_id": "pbs_capture_failure"' in ordinary_input
+ assert "Place order" in ordinary_input
+ assert "Continue to payment" in ordinary_input
+ assert '"screenshot_capture_failed"' in ordinary_input
+
+
+def test_unavailable_page_capture_is_a_typed_omission_without_invented_page_state() -> None:
+ result: dict[str, object] = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_page_unavailable",
+ "overall_status": "failed",
+ "failure_type": "runtime_error",
+ "requested_block_labels": ["continue_to_payment"],
+ "executed_block_labels": ["continue_to_payment"],
+ "blocks": [
+ {
+ "label": "continue_to_payment",
+ "status": "failed",
+ "failure_reason": "Click did not reach payment options.",
+ }
+ ],
+ "post_run_page_capture": {"status": "unavailable", "omission": "page_capture_unavailable"},
+ },
+ }
+
+ replayed_outcomes_and_packets = []
+ for _ in range(3):
+ outcome = recorded_outcome_from_run_blocks_result(result)
+ assert outcome is not None
+ packet = build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
+ replayed_outcomes_and_packets.append((outcome, packet))
+ assert len({outcome.model_dump_json() for outcome, _ in replayed_outcomes_and_packets}) == 1
+ assert len({packet.model_dump_json() for _, packet in replayed_outcomes_and_packets}) == 1
+ outcome, packet = replayed_outcomes_and_packets[0]
+
+ assert outcome.page_capture is not None
+ assert outcome.page_capture.status == "unavailable"
+ assert outcome.page_capture.omission == "page_capture_unavailable"
+ assert packet.page_capture == outcome.page_capture
+ assert packet.failure is not None and packet.failure.page_state is None
+ assert packet.page_state is None
+ assert "success_verdict" not in packet.model_dump_json()
+ assert "repair_decision" not in packet.model_dump_json()
+ assert "terminal" not in packet.model_dump_json()
+
+
+def test_stale_page_evidence_cannot_infer_a_capture_for_the_current_run() -> None:
+ result: dict[str, object] = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "browser_session_id": "pbs_current",
+ "overall_status": "failed",
+ "failure_type": "runtime_error",
+ "blocks": [{"label": "continue", "status": "failed", "failure_reason": "Click failed."}],
+ },
+ }
+ stale_scout_packet = {
+ "source_browser_session_id": "pbs_current",
+ "observed_after_workflow_run": False,
+ "forms": [{"submit_controls": [{"text": "Place order", "disabled": False}]}],
+ }
+
+ outcome = recorded_outcome_from_run_blocks_result(result, page_evidence=stale_scout_packet)
+ assert outcome is not None
+ packet = build_test_evidence_packet(_locator_packet_ctx(), result, recorded_outcome=outcome)
+
+ assert outcome.page_capture is None
+ assert outcome.page_evidence_refs == []
+ assert packet.page_capture is None
+
+
+def test_required_input_unbound_outcome_preserves_the_typed_capture_fact() -> None:
+ result: dict[str, object] = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_unbound",
+ "overall_status": "failed",
+ "blocks": [{"label": "continue", "status": "failed", "failure_reason": "Input was not bound."}],
+ "post_run_page_capture": {"status": "unavailable", "omission": "page_capture_unavailable"},
+ },
+ }
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ unbound_required_parameter_keys=["checkout_id"],
+ block_parameter_keys={"continue": ["checkout_id"]},
+ )
+
+ assert outcome is not None
+ assert outcome.reason_code == "required_input_unbound"
+ assert outcome.page_capture is not None
+ assert outcome.page_capture.model_dump() == {"status": "unavailable", "omission": "page_capture_unavailable"}
+
+
+ "return_path",
+ [
+ "terminal_challenge",
+ "demonstrated",
+ "not_evaluated",
+ "no_structural_identity",
+ "degraded_floor",
+ "recorded_failed_block",
+ "run_failed_block",
+ ],
+)
+def test_every_run_outcome_return_path_preserves_the_typed_capture_fact(return_path: str) -> None:
+ result: dict[str, object] = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_capture_paths",
+ "overall_status": "failed",
+ "failure_type": "runtime_error",
+ "blocks": [{"label": "continue", "status": "failed", "failure_reason": "Click failed."}],
+ "post_run_page_capture": {"status": "unavailable", "omission": "page_capture_unavailable"},
+ },
+ }
+ recorded_run_outcome: RecordedRunOutcome | None = None
+ completion_verification: CompletionVerificationResult | None = None
+ if return_path == "terminal_challenge":
+ recorded_run_outcome = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="terminal_challenge_blocker",
+ workflow_run_id="wr_capture_paths",
+ )
+ elif return_path == "demonstrated":
+ result = {**result, "ok": True, "data": {**result["data"], "blocks": []}}
+ recorded_run_outcome = RecordedRunOutcome(verdict="demonstrated", workflow_run_id="wr_capture_paths")
+ elif return_path == "not_evaluated":
+ result = {**result, "ok": True, "data": {**result["data"], "blocks": []}}
+ recorded_run_outcome = RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_capture_paths")
+ elif return_path == "no_structural_identity":
+ result = {**result, "ok": True, "data": {**result["data"], "blocks": [], "failure_type": None}}
+ recorded_run_outcome = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="blocker_reported",
+ workflow_run_id="wr_capture_paths",
+ )
+ elif return_path == "degraded_floor":
+ result = {**result, "ok": True, "data": {**result["data"], "blocks": [], "failure_type": None}}
+ recorded_run_outcome = RecordedRunOutcome(verdict="not_demonstrated", workflow_run_id="wr_capture_paths")
+ completion_verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="no_evidence")],
+ degraded_criterion_ids=["c0"],
+ )
+ elif return_path == "recorded_failed_block":
+ recorded_run_outcome = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="blocker_reported",
+ workflow_run_id="wr_capture_paths",
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=recorded_run_outcome,
+ completion_verification=completion_verification,
+ )
+
+ assert outcome is not None
+ assert outcome.page_capture is not None
+ assert outcome.page_capture.model_dump() == {"status": "unavailable", "omission": "page_capture_unavailable"}
+
+
+def test_page_capture_rejects_an_unavailable_page_without_its_typed_omission() -> None:
+ with pytest.raises(ValidationError, match="page_capture_unavailable"):
+ BuildTestEvidencePacket.model_validate(
+ {
+ "canonical_workflow_source": "unavailable",
+ "run": {},
+ "screenshot": {"present": False},
+ "page_capture": {"status": "unavailable"},
+ }
+ )
+
+
def test_locator_observations_reach_the_failure_packet() -> None:
result = _failed_run_result([{"authored_selector": "button.old", "match_count": 0}])
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 466667ee3..25e3e296b 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -2417,11 +2417,10 @@ def test_store_post_run_page_evidence_refuses_a_packet_read_from_another_browser
assert "workflow_run_id" not in stored
assert any(entry["event"] == _SOURCE_MISMATCH_EVENT for entry in logs)
- same_session_twin = recorded_outcome_from_run_blocks_result(
+ refused = recorded_outcome_from_run_blocks_result(
_cross_session_run_result(), page_evidence={**stored, "source_browser_session_id": "pbs_run"}
)
- assert same_session_twin is not None
- assert any("example.test" in ref for ref in same_session_twin.page_evidence_refs)
+ assert refused is None
def test_unknown_run_session_grants_a_foreign_packet_so_producers_must_stamp_the_run_session() -> None:
@@ -2668,7 +2667,7 @@ async def test_bounded_seam_capture_is_stored_stamped_without_touching_budget(
monkeypatch.setattr(composition_capture_module, "_capture_composition_evidence", fake_capture)
- await run_execution_module._capture_and_store_post_run_page(
+ capture = await run_execution_module._capture_and_store_post_run_page(
ctx, run_session_id="run_session", run_id="wr_failed", current_url="https://example.test/app/results"
)
@@ -2677,10 +2676,58 @@ async def test_bounded_seam_capture_is_stored_stamped_without_touching_budget(
assert evidence["workflow_run_id"] == "wr_failed"
assert evidence["observed_after_workflow_run"] is True
assert post_run_inspection_cleanly_matches(evidence, "wr_failed")
+ assert capture.status == "captured"
assert ctx.page_inspection_calls_this_turn == 0
assert ctx.browser_session_id is None
+async def test_post_run_capture_refused_for_a_foreign_session_reports_unavailable(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+
+ async def fake_read(
+ _ctx: CopilotContext, *, run_session_id: str, current_url: str
+ ) -> tuple[dict[str, object], str, None, None]:
+ return _bounded_failure_page_evidence(), "foreign_session", None, None
+
+ monkeypatch.setattr(run_execution_module, "_read_run_session_page_evidence", fake_read)
+
+ capture = await run_execution_module._capture_and_store_post_run_page(
+ ctx, run_session_id="run_session", run_id="wr_failed", current_url="https://example.test/app/results"
+ )
+
+ assert capture.status == "unavailable"
+ assert capture.omission == "page_capture_unavailable"
+ assert run_execution_module._same_run_page_evidence_for_result(ctx, "wr_failed") is None
+
+
+async def test_screenshot_capture_failure_keeps_a_typed_omission_on_structured_page_evidence(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _ctx()
+ monkeypatch.setattr(
+ composition_capture_module,
+ "_composition_get_screenshot",
+ AsyncMock(return_value={"ok": False, "error": "browser session unavailable"}),
+ )
+
+ evidence, captured_frame = await composition_capture_module._augment_composition_evidence_with_visual_fallback(
+ ctx,
+ {
+ "current_url": "https://example.test/checkout/payment",
+ "forms": [{"submit_controls": [{"text": "Place order", "disabled": False}]}],
+ },
+ )
+
+ assert captured_frame is None
+ assert evidence["visual_capture_omissions"] == ["screenshot_capture_failed"]
+ assert evidence["forms"] == [{"submit_controls": [{"text": "Place order", "disabled": False}]}]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"stale",
diff --git a/tests/unit/test_credential_rotation.py b/tests/unit/test_credential_rotation.py
index a25a19fa6..d57bb570d 100644
--- a/tests/unit/test_credential_rotation.py
+++ b/tests/unit/test_credential_rotation.py
@@ -1088,6 +1088,44 @@ async def test_setup_workflow_run_uses_selected_rotating_credential_for_profile_
)
+def test_managed_browser_profile_digest_key_avoids_delimiter_collision() -> None:
+ # A naive "|".join(rendered_key, credential_segment) lets two different (key, credential) pairs
+ # collide onto the same digest, since rendered_key is an unrestricted Jinja render that can
+ # itself contain "|" — e.g. key renders "acct|X" with credential "Y" vs. key "acct" with
+ # credential "X|Y" both naively joining to "acct|X|Y". That would silently reunite two distinct
+ # credentials' state on one managed profile, the exact bug SKY-15192 fixes.
+ service = WorkflowService()
+ workflow = _setup_workflow_with_rotating_credential(browser_profile_key="unused")
+
+ digest_key_a, _ = service._managed_browser_profile_digest_key(workflow, {"login_cred": "Y"}, rendered_key="acct|X")
+ digest_key_b, _ = service._managed_browser_profile_digest_key(workflow, {"login_cred": "X|Y"}, rendered_key="acct")
+
+ assert digest_key_a != digest_key_b
+
+
+async def test_setup_workflow_run_separates_managed_profile_by_credential_without_key_reference() -> None:
+ # SKY-15192: browser_profile_key is unset (or doesn't reference the rotating credential
+ # parameter), so before the fix every credential in the pool rendered the same empty digest and
+ # collapsed onto one shared managed profile. The selected credential must still segment the
+ # profile on its own.
+ result, mock_app, _, caught = await _attempt_setup_rotation_profile_run(
+ select_side_effect="cred_b",
+ profile_id="bp_selected",
+ browser_profile_key=None,
+ )
+
+ assert caught is None
+ assert result is not None
+ assert result.browser_profile_id == "bp_selected"
+ mock_app.DATABASE.browser_sessions.get_or_create_managed_browser_profile.assert_awaited_once_with(
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ browser_profile_key_digest=build_browser_profile_key_digest("cred_b"),
+ name="Workflow (auto-saved: cred_b)",
+ )
+
+
@pytest.mark.asyncio
async def test_keyed_setup_workflow_run_fails_when_rotation_selection_fails() -> None:
result, _, service, caught = await _attempt_setup_rotation_profile_run(