Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10889: gate credentialed code persistence on scouted submit (#6525)
Andrew Neilson40761ace1f0a
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 01620b4d1..31a89d7a8 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.output_policy import (from skyvern.forge.sdk.copilot.runtime import AgentContext, ScoutedInteractionfrom skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_endfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_span+from skyvern.forge.sdk.copilot.workflow_credential_utils import credential_params, parse_workflow_yaml, workflow_blocksfrom skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yamlfrom skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, InsecureCodeDetectedfrom skyvern.forge.sdk.workflow.models.block import CodeBlock@@ -214,6 +215,9 @@ _CODE_ARTIFACT_REQUIRED_LIST_FIELDS = ("completion_criteria","terminal_verifier_expectations",)+_CREDENTIAL_FIELD_ACCESS_RE = re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*)\.(username|password|totp)\b")+_CODE_SUBMIT_ACTION_RE = re.compile(r"\.(?:click|press)\s*\(")+_SCOUT_SUBMIT_TOOL_NAMES = frozenset({"click", "press_key"})def _code_artifact_metadata_as_tool_argument(@@ -738,6 +742,102 @@ def _artifact_string_list(value: Any) -> list[str]:return [str(item).strip() for item in value if str(item).strip()]+def _credentialed_code_block_scout_gate_errors(+ workflow_yaml: str | None,+ ctx: AgentContext,+) -> list[str]:+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:+ return []+ if not workflow_yaml:+ return []+ parsed = parse_workflow_yaml(workflow_yaml)+ if not isinstance(parsed, dict):+ return []+ workflow_definition = parsed.get("workflow_definition")+ if not isinstance(workflow_definition, dict):+ return []+ credential_params_by_key = credential_params(workflow_definition.get("parameters"))+ if not credential_params_by_key:+ return []+ scout_trajectory = getattr(ctx, "scout_trajectory", None)+ if not isinstance(scout_trajectory, list):+ scout_trajectory = []++ errors: list[str] = []+ for block in workflow_blocks(parsed):+ if _enum_or_string_name(block.get("block_type")) != BlockType.CODE.value:+ continue+ code = str(block.get("code") or "")+ if not code.strip():+ continue+ required_fields_by_credential: dict[str, set[str]] = {}+ for parameter_key, field in _CREDENTIAL_FIELD_ACCESS_RE.findall(code):+ credential_id = credential_params_by_key.get(parameter_key)+ if credential_id:+ required_fields_by_credential.setdefault(credential_id, set()).add(field)+ if not required_fields_by_credential:+ continue++ matched_fill_indexes: list[int] = []+ matched_source_urls: set[str] = set()+ missing_fields: list[str] = []+ for credential_id, required_fields in required_fields_by_credential.items():+ matched_fields: set[str] = set()+ for index, interaction in enumerate(scout_trajectory):+ if str(interaction.get("tool_name") or "").strip() != "fill_credential_field":+ continue+ if str(interaction.get("credential_id") or "").strip() != credential_id:+ continue+ field = str(interaction.get("credential_field") or "").strip()+ if field not in required_fields:+ continue+ matched_fields.add(field)+ matched_fill_indexes.append(index)+ source_url = str(interaction.get("source_url") or "").strip()+ if source_url:+ matched_source_urls.add(source_url)+ for field in sorted(required_fields - matched_fields):+ missing_fields.append(field)++ requires_submit = bool(_CODE_SUBMIT_ACTION_RE.search(code))+ missing_submit = False+ if requires_submit:+ latest_fill_index = max(matched_fill_indexes, default=-1)+ if latest_fill_index < 0:+ missing_submit = True+ else:+ missing_submit = True+ for index, interaction in enumerate(scout_trajectory):+ if index <= latest_fill_index:+ continue+ if str(interaction.get("tool_name") or "").strip() not in _SCOUT_SUBMIT_TOOL_NAMES:+ continue+ source_url = str(interaction.get("source_url") or "").strip()+ if matched_source_urls and source_url not in matched_source_urls:+ continue+ missing_submit = False+ break++ if not missing_fields and not missing_submit:+ continue++ block_label = str(block.get("label") or "").strip() or "this code block"+ requirements: list[str] = []+ if missing_fields:+ joined_fields = ", ".join(f"`{field}`" for field in missing_fields)+ requirements.append(f"successful `fill_credential_field` scouting for {joined_fields}")+ if missing_submit:+ requirements.append("a later submit action on the same page")+ requirement_text = " and ".join(requirements)+ errors.append(+ f"Code block `{block_label}` reads saved credential fields, but the current debug-browser scout "+ f"record is missing {requirement_text}. First scout the live form with `fill_credential_field` for "+ "each referenced credential field, then click the submit control or press Enter in the debug browser "+ "before retrying `update_workflow` or `update_and_run_blocks`."+ )+ return errors++async def _update_workflow(params: dict[str, Any],ctx: AgentContext,@@ -794,6 +894,15 @@ async def _update_workflow(code_rejected=bool(code_safety_errors),),}+ credential_scout_errors = _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)+ if credential_scout_errors:+ return {+ "ok": False,+ "error": "\n".join(credential_scout_errors),+ "user_facing_summary": (+ "I need to scout the saved-credential login flow in the debug browser before I can persist or run this code."+ ),+ }if allow_missing_credentials is None:allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is Trueif not allow_missing_credentials:diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex 7d0995437..7254ef548 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -71,6 +71,63 @@ def _code_only_ctx() -> CopilotContext:return ctx+def _standard_ctx() -> CopilotContext:+ ctx = _code_only_ctx()+ ctx.block_authoring_policy = BlockAuthoringPolicy.STANDARD+ return ctx+++def _credential_code_yaml(*, code: str, credential_id: str = "cred_missing") -> str:+ indented_code = textwrap.indent(textwrap.dedent(code).strip(), " " * 8)+ return (+ "title: Login with saved credential\n"+ "workflow_definition:\n"+ " parameters:\n"+ " - parameter_type: workflow\n"+ " workflow_parameter_type: credential_id\n"+ " key: login_credential\n"+ f" default_value: {credential_id}\n"+ " blocks:\n"+ " - block_type: code\n"+ " label: login_with_saved_credential\n"+ " code: |\n"+ f"{indented_code}\n"+ )+++def _credential_fill_interaction(+ field: str,+ *,+ credential_id: str = "cred_missing",+ source_url: str = "https://authenticationtest.com/totpChallenge/",+) -> dict[str, object]:+ selectors = {+ "username": "#email",+ "password": "input[type='password']",+ "totp": "#totpmfa",+ }+ typed_lengths = {"username": 20, "password": 14, "totp": 6}+ return {+ "tool_name": "fill_credential_field",+ "selector": selectors[field],+ "source_url": source_url,+ "credential_id": credential_id,+ "credential_field": field,+ "typed_length": typed_lengths[field],+ }+++def _submit_interaction(+ *,+ source_url: str = "https://authenticationtest.com/totpChallenge/",+) -> dict[str, object]:+ return {+ "tool_name": "click",+ "selector": "input[type='submit']",+ "source_url": source_url,+ }++class TestCodeSafetySeam:def test_import_in_new_code_block_is_a_seam_error(self) -> None:errors = _code_block_safety_errors(_IMPORTING_CODE_YAML, None)@@ -263,6 +320,101 @@ class TestStaleLabelSeamFlow:assert ref["source_tool"]+class TestCredentialScoutPersistGate:+ _SUBMIT_CODE_YAML = _credential_code_yaml(+ code="""+ await page.locator("#email").fill(login_credential.username)+ await page.locator("input[type='password']").fill(login_credential.password)+ await page.locator("#totpmfa").fill(login_credential.totp)+ await page.locator("input[type='submit']").click()+ await page.wait_for_load_state("load")+ """+ )+ _FILL_ONLY_CODE_YAML = _credential_code_yaml(+ code="""+ await page.locator("#email").fill(login_credential.username)+ await page.locator("input[type='password']").fill(login_credential.password)+ await page.locator("#totpmfa").fill(login_credential.totp)+ """+ )++ @pytest.mark.asyncio+ async def test_rejects_credential_submit_code_without_matching_fill_scouts(self) -> None:+ ctx = _code_only_ctx()+ ctx.scout_trajectory = []++ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)++ assert result["ok"] is False+ assert "fill_credential_field" in result["error"]+ assert "click the submit control or press Enter" in result["error"]+ assert result["user_facing_summary"] == (+ "I need to scout the saved-credential login flow in the debug browser before I can persist or run this code."+ )++ @pytest.mark.asyncio+ async def test_allows_submit_code_gate_once_matching_fills_and_submit_are_scouted(self) -> None:+ ctx = _code_only_ctx()+ ctx.scout_trajectory = [+ _credential_fill_interaction("username"),+ _credential_fill_interaction("password"),+ _credential_fill_interaction("totp"),+ _submit_interaction(),+ ]++ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)++ assert result["ok"] is False+ error_text = str(result.get("error") or "")+ assert "was not found in this organization" in error_text+ assert "fill_credential_field" not in error_text+ assert "saved-credential login flow" not in error_text++ @pytest.mark.asyncio+ async def test_submit_code_still_requires_later_submit_after_matching_fills(self) -> None:+ ctx = _code_only_ctx()+ ctx.scout_trajectory = [+ _credential_fill_interaction("username"),+ _credential_fill_interaction("password"),+ _credential_fill_interaction("totp"),+ ]++ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)++ assert result["ok"] is False+ assert "later submit action on the same page" in result["error"]+ assert "click the submit control or press Enter" in result["error"]++ @pytest.mark.asyncio+ async def test_fill_only_code_requires_matching_fill_scouts_but_not_submit(self) -> None:+ ctx = _code_only_ctx()+ ctx.scout_trajectory = [+ _credential_fill_interaction("username"),+ _credential_fill_interaction("password"),+ _credential_fill_interaction("totp"),+ ]++ result = await _update_workflow({"workflow_yaml": self._FILL_ONLY_CODE_YAML}, ctx)++ assert result["ok"] is False+ error_text = str(result.get("error") or "")+ assert "was not found in this organization" in error_text+ assert "click the submit control or press Enter" not in error_text+ assert "saved-credential login flow" not in error_text++ @pytest.mark.asyncio+ async def test_standard_mode_behavior_is_unchanged(self) -> None:+ ctx = _standard_ctx()+ ctx.scout_trajectory = []++ result = await _update_workflow({"workflow_yaml": self._SUBMIT_CODE_YAML}, ctx)++ assert result["ok"] is False+ error_text = str(result.get("error") or "")+ assert "fill_credential_field" not in error_text+ assert "saved-credential login flow" not in error_text++def test_run_id_leak_check_covers_non_numeric_ids() -> None:with pytest.raises(ValueError):assert_clean_user_facing_text("Outcome uncertain for wr_sample_123abc.")