Skip to content

Automate browser based workflows with AI

SKY-10889: gate credentialed code persistence on scouted submit (#6525)

Andrew Neilson40761ace1f0a
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 01620b4d1..31a89d7a8 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
from skyvern.forge.sdk.copilot.runtime import AgentContext, ScoutedInteraction
from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
+from skyvern.forge.sdk.copilot.workflow_credential_utils import credential_params, parse_workflow_yaml, workflow_blocks
from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, InsecureCodeDetected
from skyvern.forge.sdk.workflow.models.block import CodeBlock
@@ -214,6 +215,9 @@ _CODE_ARTIFACT_REQUIRED_LIST_FIELDS = (
"completion_criteria",
"terminal_verifier_expectations",
)
+_CREDENTIAL_FIELD_ACCESS_RE = re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*)\.(username|password|totp)\b")
+_CODE_SUBMIT_ACTION_RE = re.compile(r"\.(?:click|press)\s*\(")
+_SCOUT_SUBMIT_TOOL_NAMES = frozenset({"click", "press_key"})
def _code_artifact_metadata_as_tool_argument(
@@ -738,6 +742,102 @@ def _artifact_string_list(value: Any) -> list[str]:
return [str(item).strip() for item in value if str(item).strip()]
+def _credentialed_code_block_scout_gate_errors(
+ workflow_yaml: str | None,
+ ctx: AgentContext,
+) -> list[str]:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return []
+ if not workflow_yaml:
+ return []
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return []
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, dict):
+ return []
+ credential_params_by_key = credential_params(workflow_definition.get("parameters"))
+ if not credential_params_by_key:
+ return []
+ scout_trajectory = getattr(ctx, "scout_trajectory", None)
+ if not isinstance(scout_trajectory, list):
+ scout_trajectory = []
+
+ errors: list[str] = []
+ for block in workflow_blocks(parsed):
+ if _enum_or_string_name(block.get("block_type")) != BlockType.CODE.value:
+ continue
+ code = str(block.get("code") or "")
+ if not code.strip():
+ continue
+ required_fields_by_credential: dict[str, set[str]] = {}
+ for parameter_key, field in _CREDENTIAL_FIELD_ACCESS_RE.findall(code):
+ credential_id = credential_params_by_key.get(parameter_key)
+ if credential_id:
+ required_fields_by_credential.setdefault(credential_id, set()).add(field)
+ if not required_fields_by_credential:
+ continue
+
+ matched_fill_indexes: list[int] = []
+ matched_source_urls: set[str] = set()
+ missing_fields: list[str] = []
+ for credential_id, required_fields in required_fields_by_credential.items():
+ matched_fields: set[str] = set()
+ for index, interaction in enumerate(scout_trajectory):
+ if str(interaction.get("tool_name") or "").strip() != "fill_credential_field":
+ continue
+ if str(interaction.get("credential_id") or "").strip() != credential_id:
+ continue
+ field = str(interaction.get("credential_field") or "").strip()
+ if field not in required_fields:
+ continue
+ matched_fields.add(field)
+ matched_fill_indexes.append(index)
+ source_url = str(interaction.get("source_url") or "").strip()
+ if source_url:
+ matched_source_urls.add(source_url)
+ for field in sorted(required_fields - matched_fields):
+ missing_fields.append(field)
+
+ requires_submit = bool(_CODE_SUBMIT_ACTION_RE.search(code))
+ missing_submit = False
+ if requires_submit:
+ latest_fill_index = max(matched_fill_indexes, default=-1)
+ if latest_fill_index < 0:
+ missing_submit = True
+ else:
+ missing_submit = True
+ for index, interaction in enumerate(scout_trajectory):
+ if index <= latest_fill_index:
+ continue
+ if str(interaction.get("tool_name") or "").strip() not in _SCOUT_SUBMIT_TOOL_NAMES:
+ continue
+ source_url = str(interaction.get("source_url") or "").strip()
+ if matched_source_urls and source_url not in matched_source_urls:
+ continue
+ missing_submit = False
+ break
+
+ if not missing_fields and not missing_submit:
+ continue
+
+ block_label = str(block.get("label") or "").strip() or "this code block"
+ requirements: list[str] = []
+ if missing_fields:
+ joined_fields = ", ".join(f"`{field}`" for field in missing_fields)
+ requirements.append(f"successful `fill_credential_field` scouting for {joined_fields}")
+ if missing_submit:
+ requirements.append("a later submit action on the same page")
+ requirement_text = " and ".join(requirements)
+ errors.append(
+ f"Code block `{block_label}` reads saved credential fields, but the current debug-browser scout "
+ f"record is missing {requirement_text}. First scout the live form with `fill_credential_field` for "
+ "each referenced credential field, then click the submit control or press Enter in the debug browser "
+ "before retrying `update_workflow` or `update_and_run_blocks`."
+ )
+ return errors
+
+
async def _update_workflow(
params: dict[str, Any],
ctx: AgentContext,
@@ -794,6 +894,15 @@ async def _update_workflow(
code_rejected=bool(code_safety_errors),
),
}
+ credential_scout_errors = _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)
+ if credential_scout_errors:
+ return {
+ "ok": False,
+ "error": "\n".join(credential_scout_errors),
+ "user_facing_summary": (
+ "I need to scout the saved-credential login flow in the debug browser before I can persist or run this code."
+ ),
+ }
if allow_missing_credentials is None:
allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is True
if not allow_missing_credentials:
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 7d0995437..7254ef548 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -71,6 +71,63 @@ def _code_only_ctx() -> CopilotContext:
return ctx
+def _standard_ctx() -> CopilotContext:
+ ctx = _code_only_ctx()
+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD
+ return ctx
+
+
+def _credential_code_yaml(*, code: str, credential_id: str = "cred_missing") -> str:
+ indented_code = textwrap.indent(textwrap.dedent(code).strip(), " " * 8)
+ return (
+ "title: Login with saved credential\n"
+ "workflow_definition:\n"
+ " parameters:\n"
+ " - parameter_type: workflow\n"
+ " workflow_parameter_type: credential_id\n"
+ " key: login_credential\n"
+ f" default_value: {credential_id}\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: login_with_saved_credential\n"
+ " code: |\n"
+ f"{indented_code}\n"
+ )
+
+
+def _credential_fill_interaction(
+ field: str,
+ *,
+ credential_id: str = "cred_missing",
+ source_url: str = "https://authenticationtest.com/totpChallenge/",
+) -> dict[str, object]:
+ selectors = {
+ "username": "#email",
+ "password": "input[type='password']",
+ "totp": "#totpmfa",
+ }
+ typed_lengths = {"username": 20, "password": 14, "totp": 6}
+ return {
+ "tool_name": "fill_credential_field",
+ "selector": selectors[field],
+ "source_url": source_url,
+ "credential_id": credential_id,
+ "credential_field": field,
+ "typed_length": typed_lengths[field],
+ }
+
+
+def _submit_interaction(
+ *,
+ source_url: str = "https://authenticationtest.com/totpChallenge/",
+) -> dict[str, object]:
+ return {
+ "tool_name": "click",
+ "selector": "input[type='submit']",
+ "source_url": source_url,
+ }
+
+
class TestCodeSafetySeam:
def test_import_in_new_code_block_is_a_seam_error(self) -> None:
errors = _code_block_safety_errors(_IMPORTING_CODE_YAML, None)
@@ -263,6 +320,101 @@ class TestStaleLabelSeamFlow:
assert ref["source_tool"]
+class TestCredentialScoutPersistGate:
+ _SUBMIT_CODE_YAML = _credential_code_yaml(
+ code="""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#totpmfa").fill(login_credential.totp)
+ await page.locator("input[type='submit']").click()
+ await page.wait_for_load_state("load")
+ """
+ )
+ _FILL_ONLY_CODE_YAML = _credential_code_yaml(
+ code="""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#totpmfa").fill(login_credential.totp)
+ """
+ )
+
+ @pytest.mark.asyncio
+ async def test_rejects_credential_submit_code_without_matching_fill_scouts(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert "fill_credential_field" in result["error"]
+ assert "click the submit control or press Enter" in result["error"]
+ assert result["user_facing_summary"] == (
+ "I need to scout the saved-credential login flow in the debug browser before I can persist or run this code."
+ )
+
+ @pytest.mark.asyncio
+ async def test_allows_submit_code_gate_once_matching_fills_and_submit_are_scouted(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ _credential_fill_interaction("username"),
+ _credential_fill_interaction("password"),
+ _credential_fill_interaction("totp"),
+ _submit_interaction(),
+ ]
+
+ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
+
+ assert result["ok"] is False
+ error_text = str(result.get("error") or "")
+ assert "was not found in this organization" in error_text
+ assert "fill_credential_field" not in error_text
+ assert "saved-credential login flow" not in error_text
+
+ @pytest.mark.asyncio
+ async def test_submit_code_still_requires_later_submit_after_matching_fills(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ _credential_fill_interaction("username"),
+ _credential_fill_interaction("password"),
+ _credential_fill_interaction("totp"),
+ ]
+
+ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert "later submit action on the same page" in result["error"]
+ assert "click the submit control or press Enter" in result["error"]
+
+ @pytest.mark.asyncio
+ async def test_fill_only_code_requires_matching_fill_scouts_but_not_submit(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ _credential_fill_interaction("username"),
+ _credential_fill_interaction("password"),
+ _credential_fill_interaction("totp"),
+ ]
+
+ result = await _update_workflow({"workflow_yaml": self._FILL_ONLY_CODE_YAML}, ctx)
+
+ assert result["ok"] is False
+ error_text = str(result.get("error") or "")
+ assert "was not found in this organization" in error_text
+ assert "click the submit control or press Enter" not in error_text
+ assert "saved-credential login flow" not in error_text
+
+ @pytest.mark.asyncio
+ async def test_standard_mode_behavior_is_unchanged(self) -> None:
+ ctx = _standard_ctx()
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)
+
+ assert result["ok"] is False
+ error_text = str(result.get("error") or "")
+ assert "fill_credential_field" not in error_text
+ assert "saved-credential login flow" not in error_text
+
+
def test_run_id_leak_check_covers_non_numeric_ids() -> None:
with pytest.raises(ValueError):
assert_clean_user_facing_text("Outcome uncertain for wr_sample_123abc.")