Skip to content

Automate browser based workflows with AI

fix(SKY-9788): humanize init-time failure copy and skip wasted test on unbound credentials (#5972)

Andrew Neilson5ee75cd18cc1
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
index f060598b9..c828ace39 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
@@ -51,6 +51,9 @@ Continue iterating until one of these is true:
3. You are blocked by missing required user input (for example: missing credential, unresolved ambiguity, or required human-only value).
4. The latest run evidence reports that a user-supplied input is itself invalid (for example: requested dates are in the past, requested identifier or location does not exist, or a search returned zero matches for an exact-match user value). Stop on the first run that reports this and respond with ASK_QUESTION naming the offending value and asking the user for a corrected one. Do NOT keep retrying with different selectors, URLs, or block orderings. This branch does NOT cover transient retry-able failures (network errors, anti-bot/WAF/CAPTCHA, timeouts, element-not-found mid-flow) — handle those via the FEASIBILITY CHECK and ANTI-BOT / WAF BLOCK HANDLING guidance below.
+REQUEST POLICY: WORKFLOW CREDENTIAL INPUTS UNBOUND.
+When the request-policy summary shows `clarification_reason: workflow_credential_inputs_unbound` (set by the input guardrail when the workflow references credential parameters whose keys resolve to unset workflow inputs), the run-tool is blocked but `allow_update_workflow` remains true. Call `update_workflow` to land the user's edit, then REPLY with: `"I applied your requested change. I couldn't test the modified workflow because I couldn't find the required credentials — please add them via the Credentials UI, then I can try again."` Keep the unvalidated draft surfaced so the user can accept/reject. If you call `update_and_run_blocks`, the run step is skipped automatically (the tool result will include `data.skipped_run=true` and `data.skip_reason="workflow_credential_inputs_unbound"`); use the same REPLY framing.
+
DIRECT BROWSER TOOLS (OPTIONAL DIAGNOSTICS):
Prefer workflow-run evidence first. Use navigate_browser, get_browser_screenshot, evaluate, click, type_text, scroll, select_option, press_key, and console_messages only when:
1. The user explicitly asks to inspect or interact with the live page.
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 9996c9805..5c8d7b17e 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -463,6 +463,24 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> st
has_keepable_draft = ctx.last_workflow is not None and bool(ctx.last_workflow_yaml)
keep_draft_affordance = " Keep the draft to iterate on, or discard." if has_keepable_draft else ""
+ policy = ctx.request_policy if isinstance(ctx.request_policy, RequestPolicy) else None
+ if (
+ policy is not None
+ and policy.clarification_reason == "workflow_credential_inputs_unbound"
+ and ctx.last_workflow is not None
+ and ctx.last_update_block_count is not None
+ ):
+ if ctx.last_update_block_count <= 0:
+ draft_phrase = "a draft workflow"
+ else:
+ block_word = "block" if ctx.last_update_block_count == 1 else "blocks"
+ draft_phrase = f"a draft workflow with {ctx.last_update_block_count} {block_word}"
+ return (
+ f"I applied your requested change as {draft_phrase}. "
+ f"I couldn't test the modified workflow because I couldn't find the required credentials — "
+ f"please add them via the Credentials UI, then I can try again.{keep_draft_affordance}"
+ )
+
if ctx.last_test_ok is False and ctx.last_update_block_count is not None:
if ctx.last_update_block_count <= 0:
draft_phrase = "a draft workflow"
@@ -1036,15 +1054,17 @@ def _agent_output_to_text(agent_output: Any) -> str:
def _should_surface_untested_draft_despite_question(ctx: CopilotContext, response_type: str) -> bool:
+ if response_type != "ASK_QUESTION" or ctx.last_workflow is None or not ctx.last_workflow_yaml:
+ return False
request_policy = ctx.request_policy if isinstance(ctx.request_policy, RequestPolicy) else None
+ if request_policy is None or ctx.last_test_ok is not None:
+ return False
+ if request_policy.testing_intent == "skip_test" and ctx.allow_untested_workflow_draft:
+ return True
return (
- response_type == "ASK_QUESTION"
- and request_policy is not None
- and request_policy.testing_intent == "skip_test"
- and ctx.allow_untested_workflow_draft
- and ctx.last_workflow is not None
- and bool(ctx.last_workflow_yaml)
- and ctx.last_test_ok is None
+ request_policy.clarification_reason == "workflow_credential_inputs_unbound"
+ and not request_policy.allow_run_blocks
+ and request_policy.allow_missing_credentials_in_draft
)
@@ -1073,10 +1093,14 @@ def _evaluate_copilot_final_output_policy(
updated_workflow_for_kind = (
ctx.last_workflow if ctx.last_workflow is not None else WORKFLOW_PRESENT_SENTINEL if workflow_yaml else None
)
- output_kind = (
- CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL
- if surface_untested_draft
- else derive_output_kind(
+ if surface_untested_draft:
+ output_kind = (
+ CopilotOutputKind.WORKFLOW_UPDATE_PROPOSAL
+ if ctx.workflow_persisted
+ else CopilotOutputKind.WORKFLOW_DRAFT_PROPOSAL
+ )
+ else:
+ output_kind = derive_output_kind(
response_type=response_type,
request_policy=ctx.request_policy,
updated_workflow=updated_workflow_for_kind,
@@ -1084,7 +1108,6 @@ def _evaluate_copilot_final_output_policy(
workflow_attempted=workflow_attempted,
unvalidated=False,
)
- )
verdict = evaluate_output_policy(
request_policy=ctx.request_policy,
response_type=policy_response_type,
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 7f301fb85..ca47d4e90 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -15,7 +15,9 @@ from skyvern.forge.sdk.copilot.context import StructuredContext
from skyvern.forge.sdk.copilot.output_utils import parse_final_response
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.schemas.credentials import Credential
+from skyvern.forge.sdk.workflow.models.parameter import ParameterType
from skyvern.utils.strings import escape_code_fences
+from skyvern.utils.yaml_loader import safe_load_no_dates
LOG = structlog.get_logger()
PROMPT_NAME = "workflow-copilot-request-policy"
@@ -29,6 +31,7 @@ ClarificationReason = Literal[
"ambiguous_loop_edit",
"missing_conditional_condition",
"missing_target_context",
+ "workflow_credential_inputs_unbound",
]
_VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_args(ClarificationReason))
_PRE_RESOLUTION_CLARIFICATION_REASONS = {
@@ -49,6 +52,23 @@ _RAW_SECRET_QUESTION = (
_SAVED_CREDENTIAL_NAME_QUESTION = "Which saved credential should I use? Please provide the exact credential name or a credential ID beginning with cred_."
_STORED_CREDENTIAL_URL_QUESTION = "Which website or login page should I use to look up the stored credential?"
_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
+_JINJA_TEMPLATE_VAR_RE = re.compile(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}")
+_CREDENTIAL_PARAM_METADATA_FIELDS = frozenset(
+ {
+ "parameter_type",
+ "key",
+ "description",
+ "workflow_id",
+ "created_at",
+ "modified_at",
+ "deleted_at",
+ }
+)
+_LOGIN_CREDENTIAL_REQUIRED_KEY_FIELDS = ("username_key", "password_key")
+_WORKFLOW_CREDENTIAL_INPUTS_UNBOUND_QUESTION = (
+ "I couldn't find the required credentials for the existing workflow. "
+ "Please add them via the Credentials UI and I can try again."
+)
_RAW_SECRET_PATTERNS = (
re.compile(r"\b(?:password|passcode|api[_ -]?key|secret|token|bearer|authorization)\s*[:=]\s*\S+", re.I),
re.compile(
@@ -323,6 +343,8 @@ def _clarification_question(policy: RequestPolicy) -> str:
if policy.credential_input_kind == "website_stored_credential":
return _STORED_CREDENTIAL_URL_QUESTION
return "Which page or URL should the workflow go to?"
+ if policy.clarification_reason == "workflow_credential_inputs_unbound":
+ return _WORKFLOW_CREDENTIAL_INPUTS_UNBOUND_QUESTION
if policy.credential_input_kind == "credential_name":
return _SAVED_CREDENTIAL_NAME_QUESTION
if policy.credential_input_kind == "website_stored_credential":
@@ -456,6 +478,107 @@ async def _resolve_credentials(policy: RequestPolicy, organization_id: str) -> N
)
+def _is_login_credential_param(param: dict[str, Any]) -> bool:
+ raw = param.get("parameter_type")
+ if not isinstance(raw, str):
+ return False
+ try:
+ return ParameterType(raw).is_login_credential()
+ except (ValueError, TypeError):
+ return False
+
+
+def _iter_login_credential_params(parsed: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]:
+ workflow_def = parsed.get("workflow_definition")
+ if not isinstance(workflow_def, dict):
+ return []
+ out: list[tuple[str, dict[str, Any]]] = []
+ for param in workflow_def.get("parameters") or []:
+ if isinstance(param, dict) and _is_login_credential_param(param):
+ out.append(("workflow", param))
+
+ def _walk(block_list: Any) -> None:
+ if not isinstance(block_list, list):
+ return
+ for block in block_list:
+ if not isinstance(block, dict):
+ continue
+ label = str(block.get("label") or "<unlabeled>")
+ for param in block.get("parameters") or []:
+ if isinstance(param, dict) and _is_login_credential_param(param):
+ out.append((label, param))
+ _walk(block.get("loop_blocks"))
+
+ _walk(workflow_def.get("blocks"))
+ return out
+
+
+def _value_resolves_at_init(value: Any) -> bool:
+ if value is None:
+ return False
+ if isinstance(value, str):
+ return bool(value.strip())
+ return True
+
+
+def _workflow_credential_inputs_unbound(workflow_yaml: str) -> list[dict[str, str]]:
+ if not workflow_yaml:
+ return []
+ try:
+ parsed = safe_load_no_dates(workflow_yaml)
+ except Exception:
+ return []
+ if not isinstance(parsed, dict):
+ return []
+
+ workflow_def = parsed.get("workflow_definition")
+ if not isinstance(workflow_def, dict):
+ return []
+
+ workflow_param_bound: dict[str, bool] = {}
+ for param in workflow_def.get("parameters") or []:
+ if isinstance(param, dict) and param.get("parameter_type") == "workflow":
+ key = param.get("key")
+ if isinstance(key, str):
+ workflow_param_bound[key] = _value_resolves_at_init(param.get("default_value"))
+
+ findings: list[dict[str, str]] = []
+ for location, param in _iter_login_credential_params(parsed):
+ for field_name in _LOGIN_CREDENTIAL_REQUIRED_KEY_FIELDS:
+ value = param.get(field_name)
+ if not isinstance(value, str) or value.strip():
+ continue
+ findings.append(
+ {"location": location, "field": field_name, "missing": "<empty>", "kind": "credential_empty"}
+ )
+ for field_name, value in param.items():
+ if field_name in _CREDENTIAL_PARAM_METADATA_FIELDS:
+ continue
+ if not isinstance(value, str):
+ continue
+ for jinja_key in _JINJA_TEMPLATE_VAR_RE.findall(value):
+ if jinja_key in workflow_param_bound:
+ if not workflow_param_bound[jinja_key]:
+ findings.append(
+ {
+ "location": location,
+ "field": field_name,
+ "missing": jinja_key,
+ "kind": "credential_template_unbound",
+ }
+ )
+ else:
+ findings.append(
+ {
+ "location": location,
+ "field": field_name,
+ "missing": jinja_key,
+ "kind": "credential_template_undefined",
+ }
+ )
+ return findings
+
+
async def build_request_policy(
*,
user_message: str,
@@ -468,6 +591,12 @@ async def build_request_policy(
policy = await _classify_request(user_message, workflow_yaml, chat_history, global_llm_context, handler)
policy.raw_secret_detected = policy.raw_secret_detected or policy.credential_input_kind == "raw_secret"
_prioritize_credential_clarification(policy)
+
+ if policy.clarification_reason == "none" and not policy.raw_secret_detected:
+ if _workflow_credential_inputs_unbound(workflow_yaml):
+ policy.clarification_reason = "workflow_credential_inputs_unbound"
+ policy.allow_run_blocks = False
+ policy.allow_missing_credentials_in_draft = True
if policy.testing_intent == "skip_test":
policy.allow_run_blocks = False
if (
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 767eabb57..736b19ca8 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -806,6 +806,14 @@ def _request_policy_tool_error(ctx: AgentContext, tool_name: str) -> str | None:
if tool_name in BLOCK_RUNNING_TOOLS and not policy.allow_run_blocks:
if policy.testing_intent == "skip_test":
return "Request policy says the latest user message asked for an untested draft. Use update_workflow only."
+ if policy.clarification_reason == "workflow_credential_inputs_unbound":
+ return (
+ "Skipped test run: the existing workflow references credential parameters "
+ "whose keys point to workflow inputs that are not configured. REPLY to the user "
+ "with: 'I applied your requested change. I couldn't test the modified workflow "
+ "because I couldn't find the required credentials — please add them via the "
+ "Credentials UI, then I can try again.' Keep the unvalidated draft surfaced."
+ )
return (
"Request policy blocks block-running tools for the latest user message. "
"Ask the user for the required safe credential or clarification before testing."
@@ -3906,8 +3914,15 @@ async def update_and_run_blocks_tool(
"""
copilot_ctx = ctx.context
arguments = {"workflow_yaml": workflow_yaml, "block_labels": block_labels, "parameters": parameters or {}}
+ policy = getattr(copilot_ctx, "request_policy", None)
+ skip_run_after_update = (
+ isinstance(policy, RequestPolicy)
+ and policy.allow_update_workflow
+ and not policy.allow_run_blocks
+ and policy.clarification_reason == "workflow_credential_inputs_unbound"
+ )
policy_error = _request_policy_tool_error(copilot_ctx, "update_and_run_blocks")
- if policy_error:
+ if not skip_run_after_update and policy_error:
return json.dumps({"ok": False, "error": policy_error})
loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)
@@ -3955,6 +3970,25 @@ async def update_and_run_blocks_tool(
record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, result)
return json.dumps(result)
+ if skip_run_after_update:
+ skip_message = policy_error or "Skipped test run: required credentials are not configured."
+ skip_result = {
+ "ok": False,
+ "error": skip_message,
+ "data": {
+ "block_count": copilot_ctx.last_update_block_count,
+ "workflow_updated": True,
+ "skipped_run": True,
+ "skip_reason": "workflow_credential_inputs_unbound",
+ },
+ }
+ record_tool_step_result_for_ctx(copilot_ctx, "update_and_run_blocks", arguments, skip_result)
+ LOG.info(
+ "update_and_run_blocks skipped run on unbound credential workflow inputs",
+ workflow_permanent_id=copilot_ctx.workflow_permanent_id,
+ )
+ return json.dumps(skip_result)
+
# Step 2: Compute frontier and run the blocks
new_definition = None
if copilot_ctx.last_workflow is not None:
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 39a72ba60..f1f2c805f 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -158,6 +158,46 @@ class TestFailedTestResponseNormalization:
assert "not been verified end-to-end" in rewritten
assert "successful" not in rewritten.lower()
+ def test_should_surface_untested_draft_fires_on_workflow_credential_inputs_unbound(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _should_surface_untested_draft_despite_question
+ from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+
+ ctx = _ctx(
+ last_workflow=object(),
+ last_workflow_yaml="title: drafted",
+ last_test_ok=None,
+ request_policy=RequestPolicy(
+ clarification_reason="workflow_credential_inputs_unbound",
+ allow_run_blocks=False,
+ allow_missing_credentials_in_draft=True,
+ ),
+ )
+
+ assert _should_surface_untested_draft_despite_question(ctx, "ASK_QUESTION") is True
+ assert _should_surface_untested_draft_despite_question(ctx, "REPLY") is False
+
+ def test_rewrite_uses_credential_framing_when_policy_flags_unbound_inputs(self) -> None:
+ from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response
+ from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+
+ ctx = _ctx(
+ last_workflow=object(),
+ last_workflow_yaml="title: drafted",
+ last_update_block_count=12,
+ last_test_ok=None,
+ request_policy=RequestPolicy(
+ clarification_reason="workflow_credential_inputs_unbound",
+ allow_run_blocks=False,
+ allow_missing_credentials_in_draft=True,
+ ),
+ )
+ rewritten = _rewrite_failed_test_response("agent text", ctx)
+
+ assert rewritten.startswith("I applied your requested change as a draft workflow with 12 blocks.")
+ assert "I couldn't find the required credentials" in rewritten
+ assert "add them via the Credentials UI" in rewritten
+ assert "Keep the draft to iterate on, or discard." in rewritten
+
def test_rewrite_appends_keep_draft_affordance_when_draft_on_hand(self) -> None:
from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response
diff --git a/tests/unit/test_copilot_agent_prompt_chat_history_resolution.py b/tests/unit/test_copilot_agent_prompt_chat_history_resolution.py
index 2b40621f4..4758eb849 100644
--- a/tests/unit/test_copilot_agent_prompt_chat_history_resolution.py
+++ b/tests/unit/test_copilot_agent_prompt_chat_history_resolution.py
@@ -52,3 +52,14 @@ def test_carve_out_section_headers_present_in_prompt() -> None:
assert "DIAGNOSTIC / OBSERVATIONAL COMPLAINTS:" in rendered
assert "CREDENTIAL HANDLING - CRITICAL:" in rendered
assert "PARAMETERIZED REQUESTS WITHOUT A SAMPLE VALUE:" in rendered
+
+
+def test_workflow_credential_inputs_unbound_branch_teaches_reply_framing() -> None:
+ rendered = _render_agent_prompt()
+
+ assert "REQUEST POLICY: WORKFLOW CREDENTIAL INPUTS UNBOUND." in rendered
+ assert "`clarification_reason: workflow_credential_inputs_unbound`" in rendered
+ assert "Call `update_workflow` to land the user's edit" in rendered
+ assert "I applied your requested change. I couldn't test the modified workflow" in rendered
+ assert "add them via the Credentials UI" in rendered
+ assert 'data.skip_reason="workflow_credential_inputs_unbound"' in rendered
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
new file mode 100644
index 000000000..39b7a7515
--- /dev/null
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -0,0 +1,214 @@
+from __future__ import annotations
+
+import textwrap
+
+from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound
+
+
+def _yaml(body: str) -> str:
+ return textwrap.dedent(body).strip() + "\n"
+
+
+def test_detects_workflow_level_credential_referencing_empty_workflow_param() -> None:
+ yaml = _yaml(
+ """
+ title: example
+ workflow_definition:
+ parameters:
+ - key: username_key_vault_id
+ parameter_type: workflow
+ workflow_parameter_type: string
+ default_value: ''
+ - key: password_key_vault_id
+ parameter_type: workflow
+ workflow_parameter_type: string
+ default_value: null
+ - key: azure_credentials
+ parameter_type: azure_vault_credential
+ azure_vault_credential_parameter_id: azcp_528000000000000000
+ vault_name: skyvern-secret-store
+ username_key: '{{username_key_vault_id}}'
+ password_key: '{{password_key_vault_id}}'
+ blocks:
+ - block_type: login
+ label: login
+ parameters: []
+ """
+ )
+
+ findings = _workflow_credential_inputs_unbound(yaml)
+ kinds = {(f["location"], f["field"], f["kind"]) for f in findings}
+
+ assert ("workflow", "username_key", "credential_template_unbound") in kinds
+ assert ("workflow", "password_key", "credential_template_unbound") in kinds
+
+
+def test_resolves_when_workflow_param_has_non_empty_default_value() -> None:
+ yaml = _yaml(
+ """
+ title: example
+ workflow_definition:
+ parameters:
+ - key: username_key_vault_id
+ parameter_type: workflow
+ workflow_parameter_type: string
+ default_value: [email protected]
+ - key: password_key_vault_id
+ parameter_type: workflow
+ workflow_parameter_type: string
+ default_value: stored
+ - key: azure_credentials
+ parameter_type: azure_vault_credential
+ azure_vault_credential_parameter_id: azcp_real
+ vault_name: skyvern-secret-store
+ username_key: '{{username_key_vault_id}}'
+ password_key: '{{password_key_vault_id}}'
+ blocks: []
+ """
+ )
+
+ assert _workflow_credential_inputs_unbound(yaml) == []
+
+
+def test_jinja_reference_to_undefined_workflow_param_is_flagged() -> None:
+ yaml = _yaml(
+ """
+ title: example
+ workflow_definition:
+ parameters:
+ - key: azure_credentials
+ parameter_type: azure_vault_credential
+ azure_vault_credential_parameter_id: azcp_real
+ vault_name: skyvern-secret-store
+ username_key: '{{not_a_parameter}}'
+ password_key: literal
+ blocks: []
+ """
+ )
+
+ findings = _workflow_credential_inputs_unbound(yaml)
+
+ assert any(f["kind"] == "credential_template_undefined" and f["missing"] == "not_a_parameter" for f in findings)
+
+
+def test_empty_literal_credential_key_flagged() -> None:
+ yaml = _yaml(
+ """
+ title: example
+ workflow_definition:
+ parameters:
+ - key: azure_credentials
+ parameter_type: azure_vault_credential
+ azure_vault_credential_parameter_id: azcp_real
+ vault_name: skyvern-secret-store
+ username_key: ''
+ password_key: ' '
+ blocks: []
+ """
+ )
+
+ findings = _workflow_credential_inputs_unbound(yaml)
+
+ assert {(f["field"], f["kind"]) for f in findings} == {
+ ("username_key", "credential_empty"),
+ ("password_key", "credential_empty"),
+ }
+
+
+def test_mock_eval_keys_with_placeholder_suffix_are_not_false_flagged() -> None:
+ yaml = _yaml(
+ """
+ title: example
+ workflow_definition:
+ parameters:
+ - key: username_key_placeholder
+ parameter_type: workflow
+ workflow_parameter_type: string
+ default_value: mock-user
+ - key: password_key_placeholder
+ parameter_type: workflow
+ workflow_parameter_type: string
+ default_value: mock-pass
+ - key: azure_credentials
+ parameter_type: azure_vault_credential
+ azure_vault_credential_parameter_id: azcp_placeholder
+ vault_name: skyvern-secret-store
+ username_key: '{{ username_key_placeholder }}'
+ password_key: '{{ password_key_placeholder }}'
+ blocks: []
+ """
+ )
+
+ assert _workflow_credential_inputs_unbound(yaml) == []
+
+
+def test_block_level_credential_inside_loop_blocks_is_walked() -> None:
+ yaml = _yaml(
+ """
+ title: example
+ workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: for_loop
+ label: outer
+ loop_blocks:
+ - block_type: login
+ label: inner_login
+ parameters:
+ - parameter_type: azure_vault_credential
+ key: azure_credentials
+ azure_vault_credential_parameter_id: azcp_real
+ vault_name: skyvern-secret-store
+ username_key: '{{missing_key}}'
+ password_key: '{{missing_key}}'
+ """
+ )
+
+ findings = _workflow_credential_inputs_unbound(yaml)
+
+ assert any(
+ f["location"] == "inner_login" and f["field"] == "username_key" and f["kind"] == "credential_template_undefined"
+ for f in findings
+ )
+
+
+def test_non_login_credential_types_are_out_of_scope() -> None:
+ """Only login-credential types have username/password key fields; secret-only
+ types (AWS_SECRET, AZURE_SECRET, Bitwarden Sensitive/CreditCard) use different
+ schemas and fall outside this guardrail."""
+ yaml = _yaml(
+ """
+ title: example
+ workflow_definition:
+ parameters:
+ - key: my_secret
+ parameter_type: aws_secret
+ aws_key: '{{ some_unbound_key }}'
+ blocks: []
+ """
+ )
+
+ assert _workflow_credential_inputs_unbound(yaml) == []
+
+
+def test_workflow_without_credentials_returns_empty() -> None:
+ yaml = _yaml(
+ """
+ title: example
+ workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: navigation
+ label: visit
+ url: https://example.com/
+ navigation_goal: open
+ """
+ )
+
+ assert _workflow_credential_inputs_unbound(yaml) == []
+
+
+def test_malformed_or_empty_yaml_is_inert() -> None:
+ assert _workflow_credential_inputs_unbound("") == []
+ assert _workflow_credential_inputs_unbound("- not a workflow yaml\n") == []
+ assert _workflow_credential_inputs_unbound(":: broken yaml ::") == []