Skip to content

Automate browser based workflows with AI

SKY-11168: Prioritize credential blocker over code safety (#6689)

Andrew Neilson4c29a9dde67d
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 21f333f7d..b9d19b27d 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -2220,6 +2220,19 @@ async def _update_workflow(
ctx.code_artifact_metadata = merged_metadata
ctx.workflow_verification_evidence.code_artifact_metadata = merged_metadata
params["code_artifact_metadata"] = merged_metadata
+ credential_scout_errors = _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)
+ if credential_scout_errors and code_safety_errors and code_artifact_metadata_error is None:
+ return {
+ "ok": False,
+ "error": "\n".join(credential_scout_errors),
+ "user_facing_summary": (
+ "I need to verify the saved-credential login in the browser before I can save or run this code."
+ ),
+ "data": {
+ "failure_type": "missing_credential_or_init",
+ "diagnostic_code_safety_errors": code_safety_errors,
+ },
+ }
seam_errors = [error for error in (code_artifact_metadata_error, *code_safety_errors) if error]
if seam_errors:
return {
@@ -2230,14 +2243,14 @@ async def _update_workflow(
code_rejected=bool(code_safety_errors),
),
}
- credential_scout_errors = _credentialed_code_block_scout_gate_errors(workflow_yaml, ctx)
if credential_scout_errors:
return {
"ok": False,
"error": "\n".join(credential_scout_errors),
"user_facing_summary": (
- "I need to scout the saved-credential login flow in the debug browser before I can persist or run this code."
+ "I need to verify the saved-credential login in the browser before I can save or run this code."
),
+ "data": {"failure_type": "missing_credential_or_init"},
}
if allow_missing_credentials is None:
allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is True
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index af50b1bd5..f74ca94bc 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -1211,6 +1211,14 @@ class TestCredentialScoutPersistGate:
await page.locator("#totpmfa").fill(login_credential.totp)
"""
)
+ _UNSAFE_SUBMIT_CODE_YAML = _credential_code_yaml(
+ code="""
+ import asyncio
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("input[type='submit']").click()
+ """
+ )
@pytest.mark.asyncio
async def test_rejects_credential_submit_code_without_matching_fill_scouts(self) -> None:
@@ -1223,8 +1231,25 @@ class TestCredentialScoutPersistGate:
assert "fill_credential_field" in result["error"]
assert "click the submit control or press Enter" in result["error"]
assert result["user_facing_summary"] == (
- "I need to scout the saved-credential login flow in the debug browser before I can persist or run this code."
+ "I need to verify the saved-credential login in the browser before I can save or run this code."
+ )
+
+ @pytest.mark.asyncio
+ async def test_credential_scout_blocker_takes_precedence_over_code_safety(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+
+ result = await _update_workflow({"workflow_yaml": self._UNSAFE_SUBMIT_CODE_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert "fill_credential_field" in result["error"]
+ assert "Insecure code detected" not in result["error"]
+ assert result["user_facing_summary"] == (
+ "I need to verify the saved-credential login in the browser before I can save or run this code."
)
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ code_safety_diagnostics = result["data"]["diagnostic_code_safety_errors"]
+ assert any("Not allowed to import modules" in error for error in code_safety_diagnostics)
@pytest.mark.asyncio
async def test_allows_submit_code_gate_once_matching_fills_and_submit_are_scouted(self) -> None:
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 7d620a070..49266b731 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
)
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
+from skyvern.forge.sdk.copilot.tools.workflow_update import _code_block_safety_errors
from skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockStep
@@ -1220,6 +1221,32 @@ class TestCredentialFillSynthesis:
wrapped = "async def _block(page, authtest_simple):\n" + result.code
ast.parse(wrapped)
+ def test_synthesized_credential_code_passes_persist_safety_seam(self) -> None:
+ result = synthesize_code_block(
+ [
+ self._credential_fill(),
+ self._credential_fill(selector="#passwordInput", credential_field="password"),
+ ]
+ )
+ assert result is not None
+ workflow_yaml = (
+ "title: Login with saved credential\n"
+ "workflow_definition:\n"
+ " parameters:\n"
+ " - parameter_type: workflow\n"
+ " workflow_parameter_type: credential_id\n"
+ " key: authtest_simple\n"
+ " default_value: cred_123\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: login_with_saved_credential\n"
+ " parameter_keys:\n"
+ " - authtest_simple\n"
+ " code: |\n" + "\n".join(f" {line}" for line in result.code.splitlines()) + "\n"
+ )
+
+ assert _code_block_safety_errors(workflow_yaml, None) == []
+
def test_code_block_preflight_restores_recursion_limit() -> None:
before = sys.getrecursionlimit()