Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11168: Prioritize credential blocker over code safety (#6689)
Andrew Neilson4c29a9dde67d
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 21f333f7d..b9d19b27d 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -2220,6 +2220,19 @@ async def _update_workflow(ctx.code_artifact_metadata = merged_metadatactx.workflow_verification_evidence.code_artifact_metadata = merged_metadataparams["code_artifact_metadata"] = merged_metadata+ credential_scout_errors = _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)+ if credential_scout_errors and code_safety_errors and code_artifact_metadata_error is None:+ return {+ "ok": False,+ "error": "\n".join(credential_scout_errors),+ "user_facing_summary": (+ "I need to verify the saved-credential login in the browser before I can save or run this code."+ ),+ "data": {+ "failure_type": "missing_credential_or_init",+ "diagnostic_code_safety_errors": code_safety_errors,+ },+ }seam_errors = [error for error in (code_artifact_metadata_error, *code_safety_errors) if error]if seam_errors:return {@@ -2230,14 +2243,14 @@ async def _update_workflow(code_rejected=bool(code_safety_errors),),}- credential_scout_errors = _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)if credential_scout_errors:return {"ok": False,"error": "\n".join(credential_scout_errors),"user_facing_summary": (- "I need to scout the saved-credential login flow in the debug browser before I can persist or run this code."+ "I need to verify the saved-credential login in the browser before I can save or run this code."),+ "data": {"failure_type": "missing_credential_or_init"},}if allow_missing_credentials is None:allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is Truediff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex af50b1bd5..f74ca94bc 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -1211,6 +1211,14 @@ class TestCredentialScoutPersistGate:await page.locator("#totpmfa").fill(login_credential.totp)""")+ _UNSAFE_SUBMIT_CODE_YAML = _credential_code_yaml(+ code="""+ import asyncio+ await page.locator("#email").fill(login_credential.username)+ await page.locator("input[type='password']").fill(login_credential.password)+ await page.locator("input[type='submit']").click()+ """+ )@pytest.mark.asyncioasync def test_rejects_credential_submit_code_without_matching_fill_scouts(self) -> None:@@ -1223,8 +1231,25 @@ class TestCredentialScoutPersistGate:assert "fill_credential_field" in result["error"]assert "click the submit control or press Enter" in result["error"]assert result["user_facing_summary"] == (- "I need to scout the saved-credential login flow in the debug browser before I can persist or run this code."+ "I need to verify the saved-credential login in the browser before I can save or run this code."+ )++ @pytest.mark.asyncio+ async def test_credential_scout_blocker_takes_precedence_over_code_safety(self) -> None:+ ctx = _code_only_ctx()+ ctx.scout_trajectory = []++ result = await _update_workflow({"workflow_yaml": self._UNSAFE_SUBMIT_CODE_YAML}, ctx)++ assert result["ok"] is False+ assert "fill_credential_field" in result["error"]+ assert "Insecure code detected" not in result["error"]+ assert result["user_facing_summary"] == (+ "I need to verify the saved-credential login in the browser before I can save or run this code.")+ assert result["data"]["failure_type"] == "missing_credential_or_init"+ code_safety_diagnostics = result["data"]["diagnostic_code_safety_errors"]+ assert any("Not allowed to import modules" in error for error in code_safety_diagnostics)@pytest.mark.asyncioasync def test_allows_submit_code_gate_once_matching_fills_and_submit_are_scouted(self) -> None:diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.pyindex 7d620a070..49266b731 100644--- a/tests/unit/test_copilot_code_block_synthesis.py+++ b/tests/unit/test_copilot_code_block_synthesis.py@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import ()from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata+from skyvern.forge.sdk.copilot.tools.workflow_update import _code_block_safety_errorsfrom skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockStep@@ -1220,6 +1221,32 @@ class TestCredentialFillSynthesis:wrapped = "async def _block(page, authtest_simple):\n" + result.codeast.parse(wrapped)+ def test_synthesized_credential_code_passes_persist_safety_seam(self) -> None:+ result = synthesize_code_block(+ [+ self._credential_fill(),+ self._credential_fill(selector="#passwordInput", credential_field="password"),+ ]+ )+ assert result is not None+ workflow_yaml = (+ "title: Login with saved credential\n"+ "workflow_definition:\n"+ " parameters:\n"+ " - parameter_type: workflow\n"+ " workflow_parameter_type: credential_id\n"+ " key: authtest_simple\n"+ " default_value: cred_123\n"+ " blocks:\n"+ " - block_type: code\n"+ " label: login_with_saved_credential\n"+ " parameter_keys:\n"+ " - authtest_simple\n"+ " code: |\n" + "\n".join(f" {line}" for line in result.code.splitlines()) + "\n"+ )++ assert _code_block_safety_errors(workflow_yaml, None) == []+def test_code_block_preflight_restores_recursion_limit() -> None:before = sys.getrecursionlimit()