Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Resolve credential-equal workflow inputs via placeholder tokens in the planning prompt (#8056)
LawyZheng9c6003c71e20
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex e6f07dbf3..18785065c 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -4765,6 +4765,22 @@ class ForgeAgent:) -> dict[str, Any] | list | str | None:final_navigation_payload = task.navigation_payload+ # Represent any plaintext parameter value that exactly equals a stored credential value as+ # that credential's resolvable placeholder token BEFORE the synthetic values below (the real+ # verification code and the "123456" TOTP format hint) are injected, so those synthetic+ # values can never be turned into resolvable credential tokens. Otherwise such a value would+ # be one-way-redacted to [REDACTED_SECRET] at the LLM boundary and typed verbatim; here the+ # planner never sees the raw value and the existing input path resolves the token before+ # browser input.+ if (+ task.workflow_run_id is not None+ and task.workflow_run_id in app.WORKFLOW_CONTEXT_MANAGER.workflow_run_contexts+ ):+ workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(task.workflow_run_id)+ final_navigation_payload = workflow_run_context.represent_plaintext_secrets_as_placeholders(+ final_navigation_payload+ )+current_context = skyvern_context.ensure_context()verification_code = current_context.totp_codes.get(task.task_id)if verification_code:diff --git a/skyvern/forge/sdk/workflow/context_manager.py b/skyvern/forge/sdk/workflow/context_manager.pyindex 6799b0244..cd12841fc 100644--- a/skyvern/forge/sdk/workflow/context_manager.py+++ b/skyvern/forge/sdk/workflow/context_manager.py@@ -61,7 +61,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (WorkflowParameter,WorkflowParameterType,)-from skyvern.utils.secret_redaction import collect_redactable_secret_values+from skyvern.utils.secret_redaction import collect_redactable_secret_values, is_redactable_secret_valuefrom skyvern.utils.strings import generate_random_stringfrom skyvern.utils.templating import get_missing_variables@@ -521,6 +521,53 @@ class WorkflowRunContext:"""Extract registered placeholder tokens found in *text*, longest-first."""return self._scan_placeholder_tokens(text)+ def find_secret_placeholder_for_value(self, value: object) -> str | None:+ """Return the registered placeholder token whose secret value is exactly *value*.++ Lets an ordinary parameter value that duplicates a stored credential value be shown to the+ planner as the same resolvable placeholder the credential already uses, instead of a raw+ value the LLM-boundary redactor would one-way-replace with ``[REDACTED_SECRET]`` and then+ type verbatim. Only whole-value matches qualify, so a scalar that merely contains a secret+ substring is left for the redactor.++ Only values the redactor itself would treat as secrets are eligible, so this matcher never+ drifts below the redaction floor (short or sentinel values) and never re-tokenizes a value+ that is already a registered placeholder key.+ """+ if not isinstance(value, str) or not value:+ return None+ # Runtime OTP codes are also registered as secrets but have their own resolution path; leave+ # them for it rather than re-representing them here.+ if value in self.runtime_otp_values:+ return None+ if not is_redactable_secret_value(value, self.secrets):+ return None+ for secret_id, secret_value in self.secrets.items():+ if (+ isinstance(secret_id, str)+ and secret_id.startswith(RANDOM_SECRET_ID_PREFIX)+ and isinstance(secret_value, str)+ and secret_value == value+ ):+ return secret_id+ return None++ def represent_plaintext_secrets_as_placeholders(self, payload: Any) -> Any:+ """Return a copy of *payload* with any scalar equal to a registered secret value replaced+ by that secret's resolvable placeholder token.++ Containers are rebuilt so the input is not mutated; non-matching scalars, existing+ placeholders, and non-strings are returned unchanged.+ """+ if isinstance(payload, str):+ token = self.find_secret_placeholder_for_value(payload)+ return token if token is not None else payload+ if isinstance(payload, dict):+ return {key: self.represent_plaintext_secrets_as_placeholders(item) for key, item in payload.items()}+ if isinstance(payload, list):+ return [self.represent_plaintext_secrets_as_placeholders(item) for item in payload]+ return payload+def mask_secrets_in_data(self, data: Any, mask: str = "*****") -> Any:"""Recursively replace registered secret values in data with a mask.diff --git a/skyvern/utils/secret_redaction.py b/skyvern/utils/secret_redaction.pyindex 60e5a9609..40570ec12 100644--- a/skyvern/utils/secret_redaction.py+++ b/skyvern/utils/secret_redaction.py@@ -59,7 +59,7 @@ _PLACEHOLDER_TOKEN_RE = re.compile(r"placeholder_\w+")_TOTP_SENTINEL_VALUES = frozenset({"BW_TOTP", "OP_TOTP", "AZ_TOTP"})-def _is_redactable_secret_value(value: Any, secrets: Mapping[str, Any]) -> bool:+def is_redactable_secret_value(value: Any, secrets: Mapping[str, Any]) -> bool:if not isinstance(value, str):return Falseif len(value) < MIN_SECRET_LENGTH:@@ -86,7 +86,7 @@ def collect_redactable_secret_values() -> set[str]:values: set[str] = set()for value in [*secrets.values(), *extra_values]:- if _is_redactable_secret_value(value, secrets):+ if is_redactable_secret_value(value, secrets):values.add(value)for value in otp_values:if _is_redactable_otp_value(value, secrets):diff --git a/tests/unit/test_llm_prompt_secret_redaction.py b/tests/unit/test_llm_prompt_secret_redaction.pyindex b6f2d96d2..79252301a 100644--- a/tests/unit/test_llm_prompt_secret_redaction.py+++ b/tests/unit/test_llm_prompt_secret_redaction.py@@ -1,10 +1,28 @@+import jsonfrom collections.abc import Callable+from types import SimpleNamespace+from unittest.mock import MagicMock+from skyvern.forge import app as forge_app+from skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.api.llm import api_handler_factoryfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext-from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager-from skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDER+from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager, WorkflowRunContext+from skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDER, redact_secrets_from_text+from skyvern.webeye.actions.handler import get_actual_value_of_parameter_if_secret+++def _context_with_secret(workflow_run_id: str, token: str, value: str) -> WorkflowRunContext:+ context = WorkflowRunContext(+ workflow_title="t",+ workflow_id="w",+ workflow_permanent_id="wp",+ workflow_run_id=workflow_run_id,+ aws_client=MagicMock(),+ )+ context.secrets[token] = value+ return contextdef test_redact_prompt_text_removes_real_secret_and_keeps_placeholder(monkeypatch) -> None:@@ -96,3 +114,189 @@ def test_redact_message_text_content_redacts_nested_tool_result_blocks() -> Noneassert tool_result_content[0]["text"] == f"returned {REDACTED_SECRET_PLACEHOLDER}"assert tool_result_content[1] is messages[0]["content"][0]["content"][1]assert messages[0]["content"][0]["content"][0]["text"] == "returned real-password"+++def test_build_navigation_payload_represents_credential_equal_value_as_placeholder(monkeypatch) -> None:+ """A plaintext param value that duplicates a stored credential value must reach the planner as+ the credential's resolvable placeholder token, not as a raw value the LLM-boundary redactor+ would one-way-replace with [REDACTED_SECRET] and then type verbatim.+ """+ workflow_run_id = "wr_collision"+ token = "placeholder_ab12_username"+ shared_value = "shared-cred-value-0001"++ context = _context_with_secret(workflow_run_id, token, shared_value)+ manager = WorkflowContextManager()+ manager.workflow_run_contexts[workflow_run_id] = context+ monkeypatch.setattr(forge_app, "WORKFLOW_CONTEXT_MANAGER", manager)++ task = SimpleNamespace(+ task_id="tsk_collision",+ workflow_run_id=workflow_run_id,+ navigation_payload={+ "email": shared_value,+ "credentials_1": {"username": token, "password": "placeholder_ab12_password"},+ "city": "Rivertown",+ },+ )+ agent = object.__new__(ForgeAgent)+ with skyvern_context.scoped(SkyvernContext(workflow_run_id=workflow_run_id)):+ result = agent._build_navigation_payload(task) # type: ignore[arg-type]++ assert isinstance(result, dict)+ # The duplicated value is now the resolvable token, not the raw string.+ assert result["email"] == token+ # Non-colliding values and already-tokenized credentials are untouched.+ assert result["city"] == "Rivertown"+ assert result["credentials_1"]["username"] == token+ # The task's own payload is not mutated (raw value preserved for non-prompt consumers).+ assert task.navigation_payload["email"] == shared_value+++def test_represented_value_survives_redaction_as_resolvable_token(monkeypatch) -> None:+ """Negative safety: no raw credential material reaches the live planner prompt, and the+ representation stays resolvable through the existing input path.+ """+ workflow_run_id = "wr_egress"+ token = "placeholder_cd34_username"+ shared_value = "shared-cred-value-0002"++ context = _context_with_secret(workflow_run_id, token, shared_value)+ manager = WorkflowContextManager()+ manager.workflow_run_contexts[workflow_run_id] = context+ monkeypatch.setattr(forge_app, "WORKFLOW_CONTEXT_MANAGER", manager)++ represented = context.represent_plaintext_secrets_as_placeholders({"email": shared_value})+ # Simulate the LLM-boundary redaction that runs on the live prompt text.+ redacted_prompt = redact_secrets_from_text(json.dumps(represented), {shared_value})++ assert shared_value not in redacted_prompt+ assert token in redacted_prompt+ assert REDACTED_SECRET_PLACEHOLDER not in redacted_prompt+ assert get_actual_value_of_parameter_if_secret(workflow_run_id, token) == shared_value+++def test_redactor_still_one_way_redacts_secret_without_placeholder() -> None:+ """Unrelated caller unchanged: the persistence/log/artifact boundary keeps one-way-redacting a+ raw secret that has no placeholder token to [REDACTED_SECRET].+ """+ assert redact_secrets_from_text("leak shared-cred-value-0003 end", {"shared-cred-value-0003"}) == (+ f"leak {REDACTED_SECRET_PLACEHOLDER} end"+ )+++def test_represent_only_replaces_whole_value_matches() -> None:+ workflow_run_id = "wr_unit"+ token = "placeholder_ef56_username"+ value = "shared-cred-value-0004"+ context = _context_with_secret(workflow_run_id, token, value)++ payload = {+ "match": value,+ "contains": f"prefix {value} suffix",+ "already_token": token,+ "other": "unrelated",+ "nested": [value, {"deep": value}, 7, None],+ }+ result = context.represent_plaintext_secrets_as_placeholders(payload)++ assert result["match"] == token+ assert result["contains"] == f"prefix {value} suffix"+ assert result["already_token"] == token+ assert result["other"] == "unrelated"+ assert result["nested"] == [token, {"deep": token}, 7, None]+ assert payload["match"] == value+++def test_find_secret_placeholder_for_value_requires_whole_value_match() -> None:+ context = _context_with_secret("wr_find", "placeholder_gh78_username", "shared-cred-value-0005")++ assert context.find_secret_placeholder_for_value("shared-cred-value-0005") == "placeholder_gh78_username"+ assert context.find_secret_placeholder_for_value("shared-cred-value-0005 extra") is None+ assert context.find_secret_placeholder_for_value("placeholder_gh78_username") is None+ assert context.find_secret_placeholder_for_value("") is None+ assert context.find_secret_placeholder_for_value(1234) is None+++def test_non_placeholder_secret_value_stays_raw_but_boundary_redactor_still_redacts() -> None:+ """Defense-in-depth layering: a value equal to a secret registered WITHOUT a placeholder_ key is+ not tokenized here (there is no resolvable token to use), yet the untouched LLM-boundary redactor+ still one-way-redacts it to [REDACTED_SECRET]. Tokenization is additive, not a replacement.+ """+ raw_secret = "master-secret-value-0006"+ context = WorkflowRunContext(+ workflow_title="t",+ workflow_id="w",+ workflow_permanent_id="wp",+ workflow_run_id="wr_layering",+ aws_client=MagicMock(),+ )+ context.secrets["MASTER_PASSWORD"] = raw_secret++ represented = context.represent_plaintext_secrets_as_placeholders({"field": raw_secret})+ assert represented == {"field": raw_secret}+ assert redact_secrets_from_text(json.dumps(represented), {raw_secret}) == json.dumps(+ {"field": REDACTED_SECRET_PLACEHOLDER}+ )+++def test_runtime_otp_value_is_not_represented_as_placeholder() -> None:+ """OTP codes are registered as secrets too, but keep their own resolution path — the+ credential-collision representation must leave them untouched.+ """+ context = _context_with_secret("wr_otp", "placeholder_ij90_otp", "654321")+ context.runtime_otp_values.add("654321")++ assert context.find_secret_placeholder_for_value("654321") is None+ assert context.represent_plaintext_secrets_as_placeholders({"verification_code": "654321"}) == {+ "verification_code": "654321"+ }+++def test_find_secret_placeholder_respects_redactor_numeric_length_floor() -> None:+ """The matcher must not tokenize values the redactor itself would skip. A 4-digit numeric secret+ is below MIN_NUMERIC_SECRET_LENGTH (6), so the redactor ignores it; representing it would let a+ short throwaway value (a CVV/expiry) hijack every unrelated payload value that matches.+ """+ context = _context_with_secret("wr_floor_num", "placeholder_kl12_pin", "1234")+ assert context.find_secret_placeholder_for_value("1234") is None+ assert context.represent_plaintext_secrets_as_placeholders({"cvv": "1234"}) == {"cvv": "1234"}+++def test_find_secret_placeholder_respects_min_secret_length() -> None:+ """A secret value below MIN_SECRET_LENGTH (4) is ignored by the redactor and must be ignored here."""+ context = _context_with_secret("wr_floor_min", "placeholder_mn34_x", "ab")+ assert context.find_secret_placeholder_for_value("ab") is None+++def test_find_secret_placeholder_excludes_totp_sentinel() -> None:+ """TOTP sentinel values (e.g. BW_TOTP) are explicitly non-redactable and must never be tokenized."""+ context = _context_with_secret("wr_sentinel", "placeholder_op56_totp", "BW_TOTP")+ assert context.find_secret_placeholder_for_value("BW_TOTP") is None+++def test_synthetic_totp_hint_is_never_tokenized(monkeypatch) -> None:+ """The synthetic '123456' TOTP format hint injected for the LLM must never become a resolvable+ credential token, even when a run credential's value is exactly '123456'. The representation pass+ runs before the hint is injected, so the throwaway hint stays an inert literal.+ """+ workflow_run_id = "wr_totp_order"+ context = _context_with_secret(workflow_run_id, "placeholder_pw01_password", "123456")+ monkeypatch.setattr(context, "totp_secret_value_key", lambda placeholder: "totp_key")+ monkeypatch.setattr(context, "get_original_secret_value_or_none", lambda key: "totp-seed")+ manager = WorkflowContextManager()+ manager.workflow_run_contexts[workflow_run_id] = context+ monkeypatch.setattr(forge_app, "WORKFLOW_CONTEXT_MANAGER", manager)+ monkeypatch.setattr(ForgeAgent, "_should_process_totp", lambda self, scraped_page: True, raising=False)++ task = SimpleNamespace(+ task_id="tsk_totp_order",+ workflow_run_id=workflow_run_id,+ navigation_payload={"login": {"totp": "placeholder_tt99_totp"}},+ )+ agent = object.__new__(ForgeAgent)+ with skyvern_context.scoped(SkyvernContext(workflow_run_id=workflow_run_id)):+ result = agent._build_navigation_payload(task, step=SimpleNamespace()) # type: ignore[arg-type]++ assert isinstance(result, dict)+ assert result["login"]["totp"] == "123456"