Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-12974 Copilot: stop rejecting a saved credential at author time; redact at the log and database seams (#7751)
Andrew Neilson44029daee380
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 5ab8c44ca..445cc4f7f 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -192,7 +192,7 @@ from skyvern.forge.sdk.copilot.runtime import (cache_copilot_author_time_gate_log_only_ids,)from skyvern.forge.sdk.copilot.secret_redaction import SECRET_KEYWORD_ASSIGNMENT_PATTERN-from skyvern.forge.sdk.copilot.secret_scrub import scrub_secrets_from_text+from skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_values, scrub_secrets_from_textfrom skyvern.forge.sdk.copilot.streaming_adapter import (emit_turn_start,emit_workflow_draft,@@ -4090,6 +4090,7 @@ async def _translate_to_agent_result(organization_id=organization_id,workflow_yaml=workflow_yaml,settings_fallback_yaml=ctx.last_workflow_yaml or ctx.workflow_yaml,+ credential_scrub_values=registered_scrub_values(ctx),)last_workflow_yaml = workflow_yamlexcept (yaml.YAMLError, ValidationError, BaseWorkflowHTTPException) as e:diff --git a/skyvern/forge/sdk/copilot/secret_scrub.py b/skyvern/forge/sdk/copilot/secret_scrub.pyindex 6b30390e1..6789d3d8c 100644--- a/skyvern/forge/sdk/copilot/secret_scrub.py+++ b/skyvern/forge/sdk/copilot/secret_scrub.py@@ -24,6 +24,24 @@ _SESSION_SCRUB_VALUES: dict[str, list[str]] = {}# No deterministic per-session teardown exists, so FIFO-evict to bound worker memory._MAX_SCRUB_SESSIONS = 1024+# Substring-replacing a short value into stored content corrupts it: a six-digit OTP occurs by+# chance inside ports, counts, and ids. Rewrites that persist apply this floor; log redaction does+# not, because over-redacting a log line is cheap and leaking a secret into one is not.+MIN_PERSISTED_REDACTION_LENGTH = 8++_ALL_VALUES_CACHE: tuple[tuple[int, int], list[str]] | None = None+++def _registry_fingerprint() -> tuple[int, int]:+ """Cheap key that changes on any registry mutation, including a direct one.++ Values are append-only and deduped per session, so the total count moves whenever a value is+ added and the session count moves whenever one is added or dropped. Deriving the key from the+ data rather than a hand-maintained counter means a caller that mutates the dict directly cannot+ be served a stale list.+ """+ return len(_SESSION_SCRUB_VALUES), sum(len(values) for values in _SESSION_SCRUB_VALUES.values())+def _session_id(ctx: AgentContext) -> str | None:session_id = getattr(ctx, "browser_session_id", None)@@ -52,6 +70,32 @@ def clear_session_scrub_values(session_id: str | None) -> None:_SESSION_SCRUB_VALUES.pop(session_id, None)+def all_registered_secret_values() -> list[str]:+ """Every credential value registered by any session in this process, longest first.++ The log seam has no ``AgentContext`` to scope against — a credential value must never reach+ log output regardless of which session filled it. Callers that rewrite persisted content must+ use ``registered_scrub_values`` instead: replacing across sessions there would let one session's+ short value corrupt another's stored data.+ """+ global _ALL_VALUES_CACHE+ fingerprint = _registry_fingerprint()+ cached = _ALL_VALUES_CACHE+ if cached is not None and cached[0] == fingerprint:+ return cached[1]+ merged: set[str] = set()+ for values in _SESSION_SCRUB_VALUES.values():+ merged.update(value for value in values if isinstance(value, str) and value)+ computed = sorted(merged, key=len, reverse=True)+ _ALL_VALUES_CACHE = (fingerprint, computed)+ return computed+++def registered_scrub_values(ctx: AgentContext) -> list[str]:+ """This turn's and this session's registered values, longest first."""+ return _registered_scrub_values(ctx)++def _registered_scrub_values(ctx: AgentContext) -> list[str]:merged: list[str] = []values = getattr(ctx, "secret_scrub_values", None)diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.pyindex 4e588ebf1..fe6364179 100644--- a/skyvern/forge/sdk/copilot/tools/guardrails.py+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py@@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.credential_literal_rebind import (from skyvern.forge.sdk.copilot.enforcement import _record_code_authoring_guardrail_rejectfrom skyvern.forge.sdk.copilot.loop_detection import record_consecutive_tool_result_boundary_for_ctxfrom skyvern.forge.sdk.copilot.output_policy import (+ OutputPolicyReason,OutputPolicyVerdict,evaluate_output_policy,format_output_policy_tool_error,@@ -96,6 +97,11 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Toolworkflow_yaml=effective_yaml,tool_arguments=tool_arguments or raw_arguments,)+ # The credential here is the user's own, bound for the site they named, inside their own workflow,+ # so a raw value on an authoring argument is not grounds to kill the turn. Enforcement for it lives+ # at the log seam (`redact_registered_secrets`) and the persistence seam; final-output surfaces+ # still hard-block on RAW_SECRET_LEAK via `_FINAL_OUTPUT_HARD_BLOCK_REASONS`.+ verdict.remove(OutputPolicyReason.RAW_SECRET_LEAK)trace_data = output_policy_verdict_to_trace_data(verdict,surface="tool_input",@@ -103,28 +109,10 @@ def _workflow_yaml_output_policy_guardrail(data: ToolInputGuardrailData) -> Tool)residual_selectors = rebind_result.residual_selectors if rebind_result is not None else ()if verdict.allowed and residual_selectors:- # A scouted credential selector still holds a non-parameter value the rebind could not neutralize,- # and the output-policy scan does not catch every such shape. Fail closed rather than persist it.- trace_data = {- **trace_data,- "allowed": False,- "residual_raw_credential_fill_selectors": list(residual_selectors),- }- LOG.info("copilot output policy credential residual raw fill fail-closed", **trace_data)- error = (- "A credential value could not be safely bound to a workflow parameter for selector(s) "- f"{', '.join(residual_selectors)}. Fill these fields with `<credential_key>.username` / "- "`<credential_key>.password` (or `await <credential_key>.otp()`) rather than an inline value."- )- tool_name = getattr(tool_context, "tool_name", None)- if isinstance(tool_name, str) and tool_name:- record_consecutive_tool_result_boundary_for_ctx(- getattr(tool_context, "context", None),- tool_name,- {"ok": False, "error": error},- arguments=tool_arguments,- )- return ToolGuardrailFunctionOutput.reject_content(error, output_info=trace_data)+ # The rebind could not bind these selectors to a parameter. That used to reject the turn; the+ # persistence seam now redacts anything still carrying a credential, so surface it and continue.+ trace_data = {**trace_data, "residual_raw_credential_fill_selectors": list(residual_selectors)}+ LOG.warning("copilot output policy credential residual raw fill admitted", **trace_data)if verdict.allowed:if applied and rebind_result is not None:trace_data = {diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 72e7866f7..fa798b435 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -233,6 +233,7 @@ from skyvern.forge.sdk.copilot.schema_incompatibility import (merge_schema_incompatibilities,render_schema_incompatibility_agent_steer,)+from skyvern.forge.sdk.copilot.secret_scrub import registered_scrub_valuesfrom skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_endfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.turn_halt import (@@ -14635,6 +14636,7 @@ async def _update_workflow(organization_id=ctx.organization_id,workflow_yaml=workflow_yaml_with_steps,settings_fallback_yaml=prior_yaml,+ credential_scrub_values=registered_scrub_values(ctx),)_record_workflow_proxy_location_span(workflow_yaml, workflow)diff --git a/skyvern/forge/sdk/copilot/workflow_yaml.py b/skyvern/forge/sdk/copilot/workflow_yaml.pyindex 873cefa4e..af9ee7f4c 100644--- a/skyvern/forge/sdk/copilot/workflow_yaml.py+++ b/skyvern/forge/sdk/copilot/workflow_yaml.py@@ -1,5 +1,6 @@"""Copilot workflow-YAML normalization, chain repair, and Workflow conversion."""+from collections.abc import Collectionfrom datetime import datetime, timezonefrom typing import Any@@ -365,17 +366,54 @@ def _yaml_pin_saved_session_ip(workflow_yaml: str | None) -> bool | None:return _yaml_bool_setting(workflow_yaml, "pin_saved_session_ip")+def _redact_credentials_before_persistence(+ workflow_yaml: str, workflow_permanent_id: str, credential_values: Collection[str]+) -> str:+ """Last line of defence before a workflow reaches the database.++ A credential value should already have been rebound to a parameter reference upstream; reaching+ here with a live value means that failed, so redact rather than store it. The redacted workflow+ will not run as authored — that is the intended trade, and the error line is the signal to fix+ the upstream rebind.++ ``credential_values`` must be scoped to the authoring session. Replacing values registered by+ other sessions would let one org's short secret rewrite another org's stored workflow.+ """+ from skyvern.forge.sdk.copilot.secret_scrub import MIN_PERSISTED_REDACTION_LENGTH, REDACTED_SECRET_PLACEHOLDER++ redactable = {+ value for value in credential_values if isinstance(value, str) and len(value) >= MIN_PERSISTED_REDACTION_LENGTH+ }+ redacted_count = 0+ # Longest first so an overlapping shorter value never splits a longer one.+ for secret in sorted(redactable, key=len, reverse=True):+ if secret in workflow_yaml:+ redacted_count += workflow_yaml.count(secret)+ workflow_yaml = workflow_yaml.replace(secret, REDACTED_SECRET_PLACEHOLDER)+ if redacted_count:+ LOG.error(+ "copilot redacted a raw credential value at the workflow persistence seam",+ workflow_permanent_id=workflow_permanent_id,+ redacted_occurrences=redacted_count,+ )+ return workflow_yaml++async def _process_workflow_yaml(workflow_id: str,workflow_permanent_id: str,organization_id: str,workflow_yaml: str,settings_fallback_yaml: str | None = None,+ credential_scrub_values: Collection[str] = (),) -> Workflow:# Single seam every copilot YAML->Workflow conversion passes through, so code# blocks get their plain-view steps regardless of which path produced the YAML# (the update_workflow tool derives them upstream; the inline REPLACE_WORKFLOW# fallbacks would otherwise surface "No steps yet").+ workflow_yaml = _redact_credentials_before_persistence(+ workflow_yaml, workflow_permanent_id, credential_scrub_values+ )workflow_yaml = derive_code_block_steps_in_yaml(workflow_yaml)workflow_yaml_request = _normalize_copilot_yaml(workflow_yaml)diff --git a/skyvern/forge/sdk/forge_log.py b/skyvern/forge/sdk/forge_log.pyindex e2e5c8030..38cc00a41 100644--- a/skyvern/forge/sdk/forge_log.py+++ b/skyvern/forge/sdk/forge_log.py@@ -4,6 +4,7 @@ import reimport sysfrom pathlib import Pathfrom types import TracebackType+from typing import Anyimport structlogfrom structlog.typing import EventDict@@ -168,6 +169,38 @@ def add_kv_pairs_to_msg(logger: logging.Logger, method_name: str, event_dict: Evreturn event_dict+def redact_registered_secrets(logger: logging.Logger, method_name: str, event_dict: EventDict) -> EventDict:+ """Redact credential values the copilot filled during a turn from every string in the event dict.++ Imported lazily: this module is imported far earlier in boot than the copilot package.+ """+ from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER, all_registered_secret_values++ secrets = all_registered_secret_values()+ if not secrets:+ return event_dict++ def scrub(node: Any) -> Any:+ # Nested values are folded into `msg` by add_kv_pairs_to_msg further down the chain, so a+ # secret carried only inside a nested kwarg reaches the log line unless we recurse here.+ if isinstance(node, str):+ for secret in secrets:+ if secret in node:+ node = node.replace(secret, REDACTED_SECRET_PLACEHOLDER)+ return node+ if isinstance(node, dict):+ return {key: scrub(item) for key, item in node.items()}+ if isinstance(node, list):+ return [scrub(item) for item in node]+ if isinstance(node, tuple):+ return tuple(scrub(item) for item in node)+ return node++ for key, value in list(event_dict.items()):+ event_dict[key] = scrub(value)+ return event_dict++def redact_bearer_tokens(logger: logging.Logger, method_name: str, event_dict: EventDict) -> EventDict:"""Redact Bearer JWTs from any string value in the event dict.@@ -441,6 +474,7 @@ def setup_logger() -> None:additional_processors = ([redact_bearer_tokens,+ redact_registered_secrets,compact_action_objects,structlog.processors.EventRenamer("msg"),add_kv_pairs_to_msg,@@ -457,6 +491,7 @@ def setup_logger() -> None:if settings.JSON_LOGGINGelse [redact_bearer_tokens,+ redact_registered_secrets,compact_action_objects,structlog.processors.CallsiteParameterAdder({diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 58ff199ec..12b29ffd9 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -2070,7 +2070,7 @@ workflow_definition:@pytest.mark.asyncio-async def test_workflow_mutation_tools_have_sdk_input_guardrails_and_reject_raw_secret() -> None:+async def test_workflow_mutation_tools_have_sdk_input_guardrails_and_admit_raw_secret() -> None:guarded_tools = {tool.name: tool for tool in NATIVE_TOOLS if tool.name in {"update_workflow", "update_and_run_blocks"}}@@ -2082,7 +2082,7 @@ async def test_workflow_mutation_tools_have_sdk_input_guardrails_and_reject_raw_ctx.consecutive_tool_tracker = [tool_step_identity("update_workflow"), tool_step_identity("update_workflow")]ctx.failed_tool_step_tracker = {"sentinel": 2}- rejected_yaml = """+ raw_secret_yaml = """workflow_definition:blocks:- block_type: navigation@@ -2095,16 +2095,18 @@ workflow_definition:context=ctx,tool_name="update_and_run_blocks",tool_call_id="call-1",- tool_arguments=json.dumps({"workflow_yaml": rejected_yaml}),+ tool_arguments=json.dumps({"workflow_yaml": raw_secret_yaml}),),agent=SimpleNamespace(),))- assert result.behavior["type"] == "reject_content"- assert "raw_secret_leak" in result.behavior["message"]+ # A raw credential on an authoring argument is admitted here — the log and persistence seams own it —+ # so the turn survives and no failed-tool boundary is recorded.+ assert result.behavior["type"] == "allow"assert ctx.consecutive_tool_tracker == [- tool_step_identity("update_and_run_blocks", {"workflow_yaml": rejected_yaml})+ tool_step_identity("update_workflow"),+ tool_step_identity("update_workflow"),]assert ctx.failed_tool_step_tracker == {"sentinel": 2}diff --git a/tests/unit/test_copilot_secret_scrub.py b/tests/unit/test_copilot_secret_scrub.pyindex a61865af4..cb75d576e 100644--- a/tests/unit/test_copilot_secret_scrub.py+++ b/tests/unit/test_copilot_secret_scrub.py@@ -17,12 +17,16 @@ from skyvern.forge.sdk.copilot import mcp_adapter, secret_scrubfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServerfrom skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.copilot.secret_scrub import (+ MIN_PERSISTED_REDACTION_LENGTH,REDACTED_SECRET_PLACEHOLDER,+ all_registered_secret_values,clear_session_scrub_values,register_secret_scrub_value,+ registered_scrub_values,scrub_secrets_from_structure,scrub_secrets_from_text,)+from skyvern.forge.sdk.copilot.workflow_yaml import _redact_credentials_before_persistence_FAKE_PASSWORD = "fake-pa55w0rd-7x9"_FAKE_OTP = "392817"@@ -303,3 +307,72 @@ class TestCrossTurnSessionScrub:recorded_text = json.dumps(recorded)assert _FAKE_PASSWORD not in recorded_textassert _FAKE_OTP not in recorded_text+++class TestPersistenceSeam:+ """The workflow persistence seam is the last point before a credential could reach the database."""++ def test_redacts_a_registered_credential_value_carried_in_the_workflow(self) -> None:+ ctx = _agent_ctx()+ register_secret_scrub_value(ctx, _FAKE_PASSWORD)+ workflow_yaml = (+ "workflow_definition:\n"+ " blocks:\n"+ " - block_type: code\n"+ f' code: |\n await page.fill("#password", "{_FAKE_PASSWORD}")\n'+ )++ redacted = _redact_credentials_before_persistence(workflow_yaml, "wpid_1", registered_scrub_values(ctx))++ assert _FAKE_PASSWORD not in redacted+ assert REDACTED_SECRET_PLACEHOLDER in redacted++ def test_leaves_a_parameter_referenced_workflow_untouched(self) -> None:+ ctx = _agent_ctx()+ register_secret_scrub_value(ctx, _FAKE_PASSWORD)+ workflow_yaml = (+ "workflow_definition:\n"+ " blocks:\n"+ " - block_type: code\n"+ ' code: |\n await page.fill("#password", my_login.password)\n'+ )++ assert (+ _redact_credentials_before_persistence(workflow_yaml, "wpid_1", registered_scrub_values(ctx))+ == workflow_yaml+ )++ def test_another_sessions_secret_never_rewrites_this_workflow(self) -> None:+ """Scoping guard: a value registered by a different session must not touch this workflow."""+ foreign = _agent_ctx("pbs_other")+ register_secret_scrub_value(foreign, _FAKE_PASSWORD)+ authoring = _agent_ctx("pbs_mine")+ workflow_yaml = f'code: await page.fill("#note", "{_FAKE_PASSWORD}")\n'++ assert (+ _redact_credentials_before_persistence(workflow_yaml, "wpid_1", registered_scrub_values(authoring))+ == workflow_yaml+ )++ def test_a_short_value_is_not_substring_replaced(self) -> None:+ """Corruption guard: a six-digit OTP occurs by chance inside ids and counts."""+ ctx = _agent_ctx()+ register_secret_scrub_value(ctx, _FAKE_OTP)+ assert len(_FAKE_OTP) < MIN_PERSISTED_REDACTION_LENGTH+ workflow_yaml = f"code: await page.goto('https://example.com/orders/{_FAKE_OTP}')\n"++ assert (+ _redact_credentials_before_persistence(workflow_yaml, "wpid_1", registered_scrub_values(ctx))+ == workflow_yaml+ )++ def test_registered_values_are_visible_across_sessions(self) -> None:+ register_secret_scrub_value(_agent_ctx("pbs_a"), _FAKE_PASSWORD)+ register_secret_scrub_value(_agent_ctx("pbs_b"), _FAKE_OTP)++ assert set(all_registered_secret_values()) == {_FAKE_PASSWORD, _FAKE_OTP}++ def test_no_registered_values_leaves_the_workflow_alone(self) -> None:+ workflow_yaml = f'code: await page.fill("#password", "{_FAKE_PASSWORD}")\n'++ assert _redact_credentials_before_persistence(workflow_yaml, "wpid_1", ()) == workflow_yamldiff --git a/tests/unit/test_forge_log_redaction.py b/tests/unit/test_forge_log_redaction.pyindex 5c11718a7..2cab6ab97 100644--- a/tests/unit/test_forge_log_redaction.py+++ b/tests/unit/test_forge_log_redaction.py@@ -1,4 +1,24 @@-from skyvern.forge.sdk.forge_log import redact_bearer_tokens+import json+from collections.abc import Iterator++import pytest++from skyvern.forge.sdk.copilot import secret_scrub+from skyvern.forge.sdk.copilot.secret_scrub import REDACTED_SECRET_PLACEHOLDER+from skyvern.forge.sdk.forge_log import redact_bearer_tokens, redact_registered_secrets++_FAKE_CREDENTIAL = "fake-pa55w0rd-7x9"++[email protected](autouse=True)+def _isolate_session_scrub_registry() -> Iterator[None]:+ secret_scrub._SESSION_SCRUB_VALUES.clear()+ yield+ secret_scrub._SESSION_SCRUB_VALUES.clear()+++def _register_credential(value: str) -> None:+ secret_scrub._SESSION_SCRUB_VALUES.setdefault("pbs_1", []).append(value)def test_redacts_url_encoded_bearer_token() -> None:@@ -43,3 +63,54 @@ def test_redacts_in_arbitrary_string_keys() -> None:out = redact_bearer_tokens(None, "info", event) # type: ignore[arg-type]assert "abcXYZ" not in out["url"]assert out["url"].endswith("token=<redacted>")+++def test_redacts_a_registered_credential_from_a_log_event() -> None:+ _register_credential(_FAKE_CREDENTIAL)+ event = {"event": f'filling #password with "{_FAKE_CREDENTIAL}"', "selector": "#password"}+ out = redact_registered_secrets(None, "info", event) # type: ignore[arg-type]+ assert _FAKE_CREDENTIAL not in out["event"]+ assert REDACTED_SECRET_PLACEHOLDER in out["event"]+ assert out["selector"] == "#password"+++def test_redacts_a_registered_credential_from_every_string_field() -> None:+ _register_credential(_FAKE_CREDENTIAL)+ event = {"event": f"code: {_FAKE_CREDENTIAL}", "msg": f"error near {_FAKE_CREDENTIAL}"}+ out = redact_registered_secrets(None, "info", event) # type: ignore[arg-type]+ assert _FAKE_CREDENTIAL not in out["event"]+ assert _FAKE_CREDENTIAL not in out["msg"]+++def test_credential_redaction_passes_through_when_nothing_is_registered() -> None:+ event = {"event": f"contains {_FAKE_CREDENTIAL} but nothing was registered"}+ assert redact_registered_secrets(None, "info", event) == event # type: ignore[arg-type]+++def test_credential_redaction_tolerates_non_string_values() -> None:+ _register_credential(_FAKE_CREDENTIAL)+ event = {"event": "no secret here", "count": 42, "tags": ["a", "b"]}+ assert redact_registered_secrets(None, "info", event) == event # type: ignore[arg-type]+++def test_redacts_a_credential_nested_inside_a_kwarg() -> None:+ """add_kv_pairs_to_msg folds nested kwargs into the rendered line, so recursion is required."""+ _register_credential(_FAKE_CREDENTIAL)+ event = {+ "event": "tool call",+ "arguments": {"fills": [{"selector": "#pass", "value": _FAKE_CREDENTIAL}]},+ }++ out = redact_registered_secrets(None, "info", event) # type: ignore[arg-type]++ assert _FAKE_CREDENTIAL not in json.dumps(out)+ assert out["arguments"]["fills"][0]["selector"] == "#pass"+++def test_redacts_a_credential_inside_a_tuple_value() -> None:+ _register_credential(_FAKE_CREDENTIAL)+ event = {"event": "x", "pair": ("user", _FAKE_CREDENTIAL)}++ out = redact_registered_secrets(None, "info", event) # type: ignore[arg-type]++ assert out["pair"] == ("user", REDACTED_SECRET_PLACEHOLDER)