Skip to content

Automate browser based workflows with AI

test(SKY-11847): case-table the script-reviewer validator tests (118→46, coverage-identical) (#7119)

Suchintan4dc6094473d1
diff --git a/tests/unit/test_script_reviewer_quality.py b/tests/unit/test_script_reviewer_quality.py
index eadf8c468..e92cd3506 100644
--- a/tests/unit/test_script_reviewer_quality.py
+++ b/tests/unit/test_script_reviewer_quality.py
@@ -2,6 +2,8 @@
from pathlib import Path
+import pytest
+
from skyvern.core.script_generations.script_validators import validate_missing_selectors
from skyvern.services.script_reviewer import ScriptReviewer
@@ -12,17 +14,9 @@ class TestSelectorReplacementGuidance:
"""Pin Rule 8b's multi-row form guidance against accidental deletion."""
def test_rule_8b_covers_multi_row_full_block_case(self) -> None:
- # Assert on key concepts rather than exact wording so the rule can be
- # rephrased without breaking the test. The invariants are: (1) multi-row
- # forms are called out somewhere in the prompt, (2) the `full_block`
- # episode case is covered, (3) the prescribed action is removal, not
- # accumulation.
text = _PROMPT_PATH.read_text(encoding="utf-8")
assert "multi-row" in text.lower()
assert "full_block" in text
- # The fix's whole point: REMOVE (not just dedupe) the ambiguous label
- # selector. Case-sensitive — the prompt uses upper-case REMOVE as a
- # directive marker.
assert "REMOVE" in text
assert "label:has-text" in text
@@ -33,92 +27,99 @@ class TestValidateProactiveMisuse:
def setup_method(self) -> None:
self.reviewer = ScriptReviewer()
- def test_fallback_is_fine(self) -> None:
- code = """
+ @pytest.mark.parametrize(
+ ("cases",),
+ [
+ pytest.param(
+ [
+ (
+ "fallback_is_fine",
+ """
async def login(page, context):
await page.fill(selector='input[name="email"]', value='test', ai='fallback', prompt='email')
await page.click(selector='button', ai='fallback', prompt='submit')
-"""
- assert self.reviewer._validate_proactive_misuse(code) is None
-
- def test_proactive_on_fill_flagged(self) -> None:
- code = """
-async def login(page, context):
- await page.fill(selector='input[placeholder="Username"]', ai='proactive', prompt='username')
-"""
- error = self.reviewer._validate_proactive_misuse(code)
- assert error is not None
- assert "page.fill()" in error
- assert "ai='fallback'" in error
-
- def test_proactive_on_click_flagged(self) -> None:
- code = """
-async def block_fn(page, context):
- await page.click(selector='button:has-text("Submit")', ai='proactive', prompt='submit')
-"""
- error = self.reviewer._validate_proactive_misuse(code)
- assert error is not None
- assert "page.click()" in error
-
- def test_proactive_on_select_option_flagged(self) -> None:
- code = """
-async def block_fn(page, context):
- await page.select_option(selector='select[name="format"]', value='PDF', ai='proactive', prompt='format')
-"""
- error = self.reviewer._validate_proactive_misuse(code)
- assert error is not None
- assert "page.select_option()" in error
-
- def test_proactive_on_type_flagged(self) -> None:
- code = """
-async def block_fn(page, context):
- await page.type(selector='input[name="search"]', ai='proactive', prompt='search')
-"""
- error = self.reviewer._validate_proactive_misuse(code)
- assert error is not None
- assert "page.type()" in error
-
- def test_proactive_on_extract_not_flagged(self) -> None:
- """ai='proactive' on extract is legitimate — extract doesn't have selectors."""
- code = """
+""",
+ None,
+ ),
+ (
+ "proactive_on_extract_not_flagged",
+ """
async def block_fn(page, context):
result = await page.extract(prompt='Get the invoice data', ai='proactive')
-"""
- # extract is not in _INTERACTION_METHODS, so this should pass
- assert self.reviewer._validate_proactive_misuse(code) is None
-
- def test_proactive_without_selector_not_flagged(self) -> None:
- """ai='proactive' WITHOUT selector= is the documented escape hatch (SKY-9436).
-
- It is intentional and runtime-safe (always invokes LLM, no
- `selector: expected string` crash). Only flag when paired with selector=.
- """
- code = """
+""",
+ None,
+ ),
+ (
+ "proactive_without_selector_not_flagged",
+ """
async def block_fn(page, context):
await page.click(ai='proactive', prompt='Click the next-step button')
await page.fill(value='hello', ai='proactive', prompt='Fill the field')
-"""
- assert self.reviewer._validate_proactive_misuse(code) is None
-
- def test_selector_inside_prompt_does_not_falsely_flag(self) -> None:
- """Regression: prompt text containing 'selector=' must not make a proactive call look like proactive-with-selector (CORR-3)."""
- code = """
+""",
+ None,
+ ),
+ (
+ "selector_inside_prompt_does_not_falsely_flag",
+ """
async def block_fn(page, context):
await page.click(ai='proactive', prompt='No selector= available for this widget')
-"""
- assert self.reviewer._validate_proactive_misuse(code) is None
-
- def test_comments_ignored(self) -> None:
- code = """
+""",
+ None,
+ ),
+ (
+ "comments_ignored",
+ """
async def block_fn(page, context):
# await page.fill(selector='input', ai='proactive', prompt='test')
await page.fill(selector='input', value='x', ai='fallback', prompt='test')
-"""
- assert self.reviewer._validate_proactive_misuse(code) is None
-
- def test_multiline_call_flagged(self) -> None:
- """ai='proactive' on a continuation line should still be caught."""
- code = """
+""",
+ None,
+ ),
+ ],
+ id="allowed-proactive-shapes",
+ ),
+ pytest.param(
+ [
+ (
+ "proactive_on_fill_flagged",
+ """
+async def login(page, context):
+ await page.fill(selector='input[placeholder="Username"]', ai='proactive', prompt='username')
+""",
+ ("page.fill()", "ai='fallback'"),
+ ),
+ (
+ "proactive_on_click_flagged",
+ """
+async def block_fn(page, context):
+ await page.click(selector='button:has-text("Submit")', ai='proactive', prompt='submit')
+""",
+ ("page.click()",),
+ ),
+ (
+ "proactive_on_select_option_flagged",
+ """
+async def block_fn(page, context):
+ await page.select_option(selector='select[name="format"]', value='PDF', ai='proactive', prompt='format')
+""",
+ ("page.select_option()",),
+ ),
+ (
+ "proactive_on_type_flagged",
+ """
+async def block_fn(page, context):
+ await page.type(selector='input[name="search"]', ai='proactive', prompt='search')
+""",
+ ("page.type()",),
+ ),
+ ],
+ id="selector-plus-proactive-is-flagged",
+ ),
+ pytest.param(
+ [
+ (
+ "multiline_call_flagged",
+ """
async def login(page, context):
await page.fill(
selector='input[name="email"]',
@@ -126,23 +127,38 @@ async def login(page, context):
ai='proactive',
prompt='email field',
)
-"""
- error = self.reviewer._validate_proactive_misuse(code)
- assert error is not None
- assert "page.fill()" in error
-
- def test_multiple_issues_reported(self) -> None:
- code = """
+""",
+ ("page.fill()",),
+ )
+ ],
+ id="multiline-call",
+ ),
+ pytest.param(
+ [
+ (
+ "multiple_issues_reported",
+ """
async def login(page, context):
await page.fill(selector='#user', ai='proactive', prompt='user')
await page.fill(selector='#pass', ai='proactive', prompt='pass')
await page.click(selector='#submit', ai='proactive', prompt='submit')
-"""
- error = self.reviewer._validate_proactive_misuse(code)
- assert error is not None
- # Should mention multiple occurrences
- assert "page.fill()" in error
- assert "page.click()" in error
+""",
+ ("page.fill()", "page.click()"),
+ )
+ ],
+ id="multiple-issues",
+ ),
+ ],
+ )
+ def test_proactive_misuse_cases(self, cases: list[tuple[str, str, tuple[str, ...] | None]]) -> None:
+ for case_id, code, expected_substrings in cases:
+ error = self.reviewer._validate_proactive_misuse(code)
+ if expected_substrings is None:
+ assert error is None, case_id
+ continue
+ assert error is not None, case_id
+ for expected in expected_substrings:
+ assert expected in error, case_id
class TestValidateFragileSelectors:
@@ -151,98 +167,122 @@ class TestValidateFragileSelectors:
def setup_method(self) -> None:
self.reviewer = ScriptReviewer()
- def test_stable_selectors_pass(self) -> None:
- code = """
+ @pytest.mark.parametrize(
+ ("cases",),
+ [
+ pytest.param(
+ [
+ (
+ "stable_selectors_pass",
+ """
async def login(page, context):
await page.fill(selector='input[name="email"]', value='test', ai='fallback', prompt='email')
await page.click(selector='button:has-text("Sign In")', ai='fallback', prompt='sign in')
await page.fill(selector='input[placeholder="Password"]', value='pass', ai='fallback', prompt='pass')
-"""
- assert self.reviewer._validate_fragile_selectors(code) is None
-
- def test_dotnetnuke_id_flagged(self) -> None:
- code = """
+""",
+ None,
+ ),
+ (
+ "comments_ignored",
+ """
+async def block_fn(page, context):
+ # selector='#dnn_ctl00_aMyAccount' is fragile but this is a comment
+ await page.click(selector='button:has-text("Submit")', ai='fallback', prompt='submit')
+""",
+ None,
+ ),
+ (
+ "no_selector_passes",
+ """
+async def block_fn(page, context):
+ await page.click(ai='fallback', prompt='click the submit button')
+""",
+ None,
+ ),
+ ],
+ id="stable-and-ignored",
+ ),
+ pytest.param(
+ [
+ (
+ "dotnetnuke_id_flagged",
+ """
async def login(page, context):
await page.click(selector='#dnn_ctl00_aMyAccount', ai='fallback', prompt='account')
-"""
- error = self.reviewer._validate_fragile_selectors(code)
- assert error is not None
- assert "dnn_" in error
- assert "DotNetNuke" in error
-
- def test_ember_id_flagged(self) -> None:
- code = """
+""",
+ ("dnn_", "DotNetNuke"),
+ ),
+ (
+ "ember_id_flagged",
+ """
async def block_fn(page, context):
await page.click(selector='#ember-1234', ai='fallback', prompt='click')
-"""
- error = self.reviewer._validate_fragile_selectors(code)
- assert error is not None
- assert "ember" in error.lower()
-
- def test_react_select_id_flagged(self) -> None:
- code = """
+""",
+ ("ember",),
+ ),
+ (
+ "react_select_id_flagged",
+ """
async def block_fn(page, context):
await page.click(selector='#react-select-5-option-2', ai='fallback', prompt='select option')
-"""
- error = self.reviewer._validate_fragile_selectors(code)
- assert error is not None
- assert "react-select" in error.lower()
-
- def test_css_in_js_class_flagged(self) -> None:
- code = """
+""",
+ ("react-select",),
+ ),
+ (
+ "css_in_js_class_flagged",
+ """
async def block_fn(page, context):
await page.click(selector='.css-1a2b3c', ai='fallback', prompt='click button')
-"""
- error = self.reviewer._validate_fragile_selectors(code)
- assert error is not None
- assert "css-" in error.lower()
-
- def test_mui_class_flagged(self) -> None:
- code = """
+""",
+ ("css-",),
+ ),
+ (
+ "mui_class_flagged",
+ """
async def block_fn(page, context):
await page.click(selector='.MuiButton-root', ai='fallback', prompt='click button')
-"""
- error = self.reviewer._validate_fragile_selectors(code)
- assert error is not None
-
- def test_extjs_id_flagged(self) -> None:
- code = """
+""",
+ (),
+ ),
+ (
+ "extjs_id_flagged",
+ """
async def block_fn(page, context):
await page.click(selector='#ext-gen-456', ai='fallback', prompt='click')
-"""
- error = self.reviewer._validate_fragile_selectors(code)
- assert error is not None
- assert "ext-gen" in error.lower()
-
- def test_comments_ignored(self) -> None:
- code = """
-async def block_fn(page, context):
- # selector='#dnn_ctl00_aMyAccount' is fragile but this is a comment
- await page.click(selector='button:has-text("Submit")', ai='fallback', prompt='submit')
-"""
- assert self.reviewer._validate_fragile_selectors(code) is None
-
- def test_multiline_call_flagged(self) -> None:
- """Fragile selector on a continuation line should still be caught."""
- code = """
+""",
+ ("ext-gen",),
+ ),
+ ],
+ id="fragile-framework-patterns",
+ ),
+ pytest.param(
+ [
+ (
+ "multiline_call_flagged",
+ """
async def block_fn(page, context):
await page.click(
selector='#dnn_ctl00_aMyAccount',
ai='fallback',
prompt='account link',
)
-"""
- error = self.reviewer._validate_fragile_selectors(code)
- assert error is not None
- assert "dnn_" in error
-
- def test_no_selector_passes(self) -> None:
- """Code without selector= kwargs should not trigger."""
- code = """
-async def block_fn(page, context):
- await page.click(ai='fallback', prompt='click the submit button')
-"""
- assert self.reviewer._validate_fragile_selectors(code) is None
+""",
+ ("dnn_",),
+ )
+ ],
+ id="multiline-selector",
+ ),
+ ],
+ )
+ def test_fragile_selector_cases(self, cases: list[tuple[str, str, tuple[str, ...] | None]]) -> None:
+ for case_id, code, expected_substrings in cases:
+ error = self.reviewer._validate_fragile_selectors(code)
+ if expected_substrings is None:
+ assert error is None, case_id
+ continue
+ assert error is not None, case_id
+ for expected in expected_substrings:
+ assert expected.lower() in error.lower(), case_id
class TestValidateHardcodedRunData:
@@ -251,101 +291,111 @@ class TestValidateHardcodedRunData:
def setup_method(self) -> None:
self.reviewer = ScriptReviewer()
- def test_clean_code_passes(self) -> None:
- code = """
+ @pytest.mark.parametrize(
+ ("cases",),
+ [
+ pytest.param(
+ [
+ (
+ "clean_code_passes",
+ """
async def block_fn(page, context):
await page.click(selector='button:has-text("Download")', ai='fallback', prompt='download invoice')
-"""
- assert self.reviewer._validate_hardcoded_run_data(code) is None
-
- def test_date_in_selector_flagged(self) -> None:
- code = """
+""",
+ None,
+ ),
+ (
+ "stable_short_text_ok",
+ """
async def block_fn(page, context):
- await page.click(selector='a:has-text("03/17/2026")', ai='fallback', prompt='click invoice')
-"""
- error = self.reviewer._validate_hardcoded_run_data(code)
- assert error is not None
- assert "03/17/2026" in error
- assert "date" in error.lower()
-
- def test_iso_date_in_selector_flagged(self) -> None:
- code = """
+ await page.click(selector='button:has-text("OK")', ai='fallback', prompt='confirm')
+""",
+ None,
+ ),
+ (
+ "long_has_text_ok",
+ """
async def block_fn(page, context):
- await page.click(selector='td:has-text("2026-03-17")', ai='fallback', prompt='click invoice')
-"""
- error = self.reviewer._validate_hardcoded_run_data(code)
- assert error is not None
- assert "2026-03-17" in error
-
- def test_date_in_prompt_flagged(self) -> None:
- code = """
+ await page.click(selector='a:has-text("Download")', ai='fallback', prompt='download')
+""",
+ None,
+ ),
+ (
+ "comments_ignored",
+ """
async def block_fn(page, context):
- await page.select_option(selector='select', value='PDF', ai='proactive', prompt='Select format for invoice dated 3/17/2026')
-"""
- error = self.reviewer._validate_hardcoded_run_data(code)
- assert error is not None
- assert "3/17/2026" in error
- assert "prompt" in error.lower()
-
- def test_short_has_text_flagged(self) -> None:
- """a:has-text("6") is almost certainly hardcoded run data."""
- code = """
+ # Don't use selector='a:has-text("6")' — it's hardcoded
+ await page.click(selector='a:has-text("Download")', ai='fallback', prompt='download')
+""",
+ None,
+ ),
+ (
+ "parameterized_date_ok",
+ """
async def block_fn(page, context):
- await page.click(selector='a:has-text("6")', ai='fallback', prompt='click invoice')
-"""
- error = self.reviewer._validate_hardcoded_run_data(code)
- assert error is not None
- assert ':has-text("6")' in error
-
- def test_short_has_text_number_flagged(self) -> None:
- code = """
+ start_date = context.parameters['download_start_date']
+ await page.click(selector=f'a:has-text("{start_date}")', ai='fallback', prompt='click date')
+""",
+ None,
+ ),
+ (
+ "text_patterns_without_email_ok",
+ """
async def block_fn(page, context):
- await page.click(selector='a:has-text("12")', ai='fallback', prompt='click row')
-"""
- error = self.reviewer._validate_hardcoded_run_data(code)
- assert error is not None
-
- def test_stable_short_text_ok(self) -> None:
- """Common UI elements with short text like 'OK' or 'X' should pass."""
- code = """
+ state = await page.classify(
+ options={"login": "login page", "dashboard": "dashboard"},
+ text_patterns={
+ "login": "Welcome, Username, Password, Sign in",
+ "dashboard": "Logout, Billing & Payments, Service Management",
+ },
+ )
+""",
+ None,
+ ),
+ ],
+ id="allowed-run-data-shapes",
+ ),
+ pytest.param(
+ [
+ (
+ "date_in_selector_flagged",
+ """
async def block_fn(page, context):
- await page.click(selector='button:has-text("OK")', ai='fallback', prompt='confirm')
-"""
- assert self.reviewer._validate_hardcoded_run_data(code) is None
-
- def test_long_has_text_ok(self) -> None:
- """:has-text with meaningful text (3+ chars) should pass."""
- code = """
+ await page.click(selector='a:has-text("03/17/2026")', ai='fallback', prompt='click invoice')
+""",
+ ("03/17/2026", "date"),
+ ),
+ (
+ "iso_date_in_selector_flagged",
+ """
async def block_fn(page, context):
- await page.click(selector='a:has-text("Download")', ai='fallback', prompt='download')
-"""
- assert self.reviewer._validate_hardcoded_run_data(code) is None
-
- def test_comments_ignored(self) -> None:
- code = """
+ await page.click(selector='td:has-text("2026-03-17")', ai='fallback', prompt='click invoice')
+""",
+ ("2026-03-17",),
+ ),
+ (
+ "date_in_prompt_flagged",
+ """
async def block_fn(page, context):
- # Don't use selector='a:has-text("6")' — it's hardcoded
- await page.click(selector='a:has-text("Download")', ai='fallback', prompt='download')
-"""
- assert self.reviewer._validate_hardcoded_run_data(code) is None
-
- def test_multiline_date_in_selector_flagged(self) -> None:
- """Hardcoded date on a continuation line should still be caught."""
- code = """
+ await page.select_option(selector='select', value='PDF', ai='proactive', prompt='Select format for invoice dated 3/17/2026')
+""",
+ ("3/17/2026", "prompt"),
+ ),
+ (
+ "multiline_date_in_selector_flagged",
+ """
async def block_fn(page, context):
await page.click(
selector='a:has-text("03/17/2026")',
ai='fallback',
prompt='click invoice',
)
-"""
- error = self.reviewer._validate_hardcoded_run_data(code)
- assert error is not None
- assert "03/17/2026" in error
-
- def test_multiline_date_in_prompt_flagged(self) -> None:
- """Hardcoded date in prompt on a continuation line should still be caught."""
- code = """
+""",
+ ("03/17/2026",),
+ ),
+ (
+ "multiline_date_in_prompt_flagged",
+ """
async def block_fn(page, context):
await page.select_option(
selector='select',
@@ -353,24 +403,38 @@ async def block_fn(page, context):
ai='proactive',
prompt='Select format for invoice dated 3/17/2026',
)
-"""
- error = self.reviewer._validate_hardcoded_run_data(code)
- assert error is not None
- assert "3/17/2026" in error
-
- def test_parameterized_date_ok(self) -> None:
- """Dates in context.parameters references should not trigger."""
- code = """
+""",
+ ("3/17/2026",),
+ ),
+ ],
+ id="date-literals-flagged",
+ ),
+ pytest.param(
+ [
+ (
+ "short_has_text_flagged",
+ """
async def block_fn(page, context):
- start_date = context.parameters['download_start_date']
- await page.click(selector=f'a:has-text("{start_date}")', ai='fallback', prompt='click date')
-"""
- # f-string selectors won't match the date regex in the selector value
- assert self.reviewer._validate_hardcoded_run_data(code) is None
-
- def test_email_in_text_patterns_flagged(self) -> None:
- """Email addresses in text_patterns are PII that should not be in cached scripts."""
- code = """
+ await page.click(selector='a:has-text("6")', ai='fallback', prompt='click invoice')
+""",
+ (':has-text("6")',),
+ ),
+ (
+ "short_has_text_number_flagged",
+ """
+async def block_fn(page, context):
+ await page.click(selector='a:has-text("12")', ai='fallback', prompt='click row')
+""",
+ (),
+ ),
+ ],
+ id="short-has-text-flagged",
+ ),
+ pytest.param(
+ [
+ (
+ "email_in_text_patterns_flagged",
+ """
async def block_fn(page, context):
state = await page.classify(
options={"login": "login page", "dashboard": "dashboard"},
@@ -379,24 +443,23 @@ async def block_fn(page, context):
"dashboard": "Logout, [email protected], John Smith, Billing",
},
)
-"""
- error = self.reviewer._validate_hardcoded_run_data(code)
- assert error is not None
- assert "email" in error.lower()
-
- def test_text_patterns_without_email_ok(self) -> None:
- """Generic text_patterns without PII should pass."""
- code = """
-async def block_fn(page, context):
- state = await page.classify(
- options={"login": "login page", "dashboard": "dashboard"},
- text_patterns={
- "login": "Welcome, Username, Password, Sign in",
- "dashboard": "Logout, Billing & Payments, Service Management",
- },
+""",
+ ("email",),
+ )
+ ],
+ id="text-patterns-pii",
+ ),
+ ],
)
-"""
- assert self.reviewer._validate_hardcoded_run_data(code) is None
+ def test_hardcoded_run_data_cases(self, cases: list[tuple[str, str, tuple[str, ...] | None]]) -> None:
+ for case_id, code, expected_substrings in cases:
+ error = self.reviewer._validate_hardcoded_run_data(code)
+ if expected_substrings is None:
+ assert error is None, case_id
+ continue
+ assert error is not None, case_id
+ for expected in expected_substrings:
+ assert expected.lower() in error.lower(), case_id
class TestValidateMissingSelectors:
@@ -421,10 +484,13 @@ async def block_fn(page, context):
class TestExtractCachedBlocks:
"""Tests for the shared block extraction utilities."""
- def test_extract_all_blocks(self):
- from skyvern.services.workflow_script_service import extract_cached_blocks_from_source
+ def test_extract_cached_block_cases(self) -> None:
+ from skyvern.services.workflow_script_service import (
+ extract_cached_blocks_from_source,
+ extract_single_cached_block,
+ )
- source = """import skyvern
+ multi_block_source = """import skyvern
@skyvern.cached(cache_key = 'login')
async def login(page, context):
@@ -437,118 +503,74 @@ async def block_1(page, context):
await page.click(selector='button')
await page.complete()
"""
- result = extract_cached_blocks_from_source(source)
+ result = extract_cached_blocks_from_source(multi_block_source)
assert set(result.keys()) == {"login", "block_1"}
assert "page.goto" in result["login"] # nosemgrep: incomplete-url-substring-sanitization
assert "page.click" in result["block_1"] # nosemgrep: incomplete-url-substring-sanitization
- # login block should NOT contain block_1 code
assert "page.click" not in result["login"]
- def test_extract_single_block(self):
- from skyvern.services.workflow_script_service import extract_single_cached_block
-
- source = """import skyvern
-
[email protected](cache_key = 'login')
-async def login(page, context):
- await page.goto('https://example.com')
-
-
[email protected](cache_key = 'block_1')
-async def block_1(page, context):
- await page.click(selector='button')
-"""
- result = extract_single_cached_block(source, "block_1")
- assert result is not None
- assert "page.click" in result # nosemgrep: incomplete-url-substring-sanitization
- assert "page.goto" not in result
-
- def test_extract_first_block(self):
- from skyvern.services.workflow_script_service import extract_single_cached_block
-
- source = """import skyvern
-
[email protected](cache_key = 'login')
-async def login(page, context):
- await page.goto('https://example.com')
-
-
[email protected](cache_key = 'block_1')
-async def block_1(page, context):
- await page.click(selector='button')
-"""
- result = extract_single_cached_block(source, "login")
- assert result is not None
- assert "page.goto" in result # nosemgrep: incomplete-url-substring-sanitization
- assert "page.click" not in result
-
- def test_extract_single_block_not_found(self):
- from skyvern.services.workflow_script_service import extract_single_cached_block
-
- source = "@skyvern.cached(cache_key = 'login')\nasync def login(page, context): pass\n"
- result = extract_single_cached_block(source, "nonexistent")
- assert result is None
-
- def test_extract_empty_source(self):
- from skyvern.services.workflow_script_service import extract_cached_blocks_from_source
-
+ single_block = extract_single_cached_block(multi_block_source, "block_1")
+ assert single_block is not None
+ assert "page.click" in single_block # nosemgrep: incomplete-url-substring-sanitization
+ assert "page.goto" not in single_block
+
+ first_block = extract_single_cached_block(multi_block_source, "login")
+ assert first_block is not None
+ assert "page.goto" in first_block # nosemgrep: incomplete-url-substring-sanitization
+ assert "page.click" not in first_block
+
+ assert (
+ extract_single_cached_block(
+ "@skyvern.cached(cache_key = 'login')\nasync def login(page, context): pass\n",
+ "nonexistent",
+ )
+ is None
+ )
assert extract_cached_blocks_from_source("") == {}
assert extract_cached_blocks_from_source("import skyvern\n") == {}
- def test_extract_last_block_goes_to_eof(self):
- from skyvern.services.workflow_script_service import extract_cached_blocks_from_source
-
- source = "@skyvern.cached(cache_key = 'only_block')\nasync def only(page, ctx):\n pass\n"
- result = extract_cached_blocks_from_source(source)
- assert "only_block" in result
- assert "pass" in result["only_block"]
+ last_block_source = "@skyvern.cached(cache_key = 'only_block')\nasync def only(page, ctx):\n pass\n"
+ last_block = extract_cached_blocks_from_source(last_block_source)
+ assert "only_block" in last_block
+ assert "pass" in last_block["only_block"]
class TestClassifyBlockStrategy:
"""Tests for _classify_block_strategy template selection."""
- def setup_method(self):
- from skyvern.services.script_reviewer import ScriptReviewer
-
+ def setup_method(self) -> None:
self.reviewer = ScriptReviewer()
- def test_extraction_block(self):
- """Block with page.extract() and no page.click() → extraction."""
- code = "async def block(page, ctx):\n return await page.extract(prompt='...')\n"
- result = self.reviewer._classify_block_strategy(existing_code=code)
- assert result == "extraction"
-
- def test_extraction_with_click_is_sequential(self):
- """Block with both extract and click → sequential (not extraction)."""
- code = (
- "async def block(page, ctx):\n await page.click('#btn')\n return await page.extract(prompt='...')\n"
- )
- result = self.reviewer._classify_block_strategy(existing_code=code)
- assert result == "sequential"
-
- def test_existing_fill_form_stays_form_filling(self):
- """Block already using page.fill_form() stays form_filling."""
- code = "async def block(page, ctx):\n await page.fill_form(ctx.parameters, prompt='...')\n"
- result = self.reviewer._classify_block_strategy(existing_code=code)
- assert result == "form_filling"
-
- def test_download_block_is_sequential(self):
- """Download block should be sequential, never form_filling."""
- code = "async def download(page, ctx):\n await page.download_file(prompt='...')\n await page.complete()\n"
- result = self.reviewer._classify_block_strategy(existing_code=code)
- assert result == "sequential"
-
- def test_navigation_block_with_form_fields_is_sequential(self):
- """Block on a page with form fields should still be sequential (not form_filling).
-
- Regression guard: this data (5 form fields + form keywords in goal + form
- action types) would have triggered the removed heuristic. Verifies it's gone.
- """
- code = "async def block(page, ctx):\n await page.click('#btn')\n await page.complete()\n"
- result = self.reviewer._classify_block_strategy(existing_code=code)
- assert result == "sequential"
-
- def test_default_is_sequential(self):
- """Empty code, no special signals → sequential."""
- result = self.reviewer._classify_block_strategy(existing_code="")
- assert result == "sequential"
+ @pytest.mark.parametrize(
+ ("code", "expected"),
+ [
+ pytest.param(
+ "async def block(page, ctx):\n return await page.extract(prompt='...')\n",
+ "extraction",
+ id="extract-only",
+ ),
+ pytest.param(
+ "async def block(page, ctx):\n await page.click('#btn')\n return await page.extract(prompt='...')\n",
+ "sequential",
+ id="extract-with-click",
+ ),
+ pytest.param(
+ "async def block(page, ctx):\n await page.fill_form(ctx.parameters, prompt='...')\n",
+ "form_filling",
+ id="existing-fill-form",
+ ),
+ pytest.param(
+ "async def download(page, ctx):\n await page.download_file(prompt='...')\n await page.complete()\n",
+ "sequential",
+ id="download",
+ ),
+ pytest.param(
+ "async def block(page, ctx):\n await page.click('#btn')\n await page.complete()\n",
+ "sequential",
+ id="navigation-with-form-fields",
+ ),
+ pytest.param("", "sequential", id="default"),
+ ],
+ )
+ def test_classify_block_strategy_cases(self, code: str, expected: str) -> None:
+ assert self.reviewer._classify_block_strategy(existing_code=code) == expected
diff --git a/tests/unit/test_script_reviewer_validators.py b/tests/unit/test_script_reviewer_validators.py
index 8568bfa7c..4add991cc 100644
--- a/tests/unit/test_script_reviewer_validators.py
+++ b/tests/unit/test_script_reviewer_validators.py
@@ -1,11 +1,17 @@
"""Tests for ScriptReviewer validation methods."""
from datetime import datetime
+from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
+from skyvern.core.script_generations.generate_script import (
+ MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB,
+ MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB,
+)
from skyvern.forge import app
+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterType
from skyvern.schemas.scripts import ScriptFallbackEpisode
from skyvern.services.script_reviewer import ScriptReviewer
from tests.unit.force_stub_app import start_forge_stub_app
@@ -17,320 +23,286 @@ class TestValidateNoHardcodedValues:
def setup_method(self) -> None:
self.reviewer = ScriptReviewer()
- def test_no_params_returns_none(self) -> None:
- code = 'await page.click(selector="button")'
- assert self.reviewer._validate_no_hardcoded_values(code, None) is None
- assert self.reviewer._validate_no_hardcoded_values(code, {}) is None
-
- def test_short_values_ignored(self) -> None:
- """Values shorter than 5 chars should not trigger (too many false positives)."""
- code = """
+ @pytest.mark.parametrize(
+ ("cases",),
+ [
+ pytest.param(
+ [
+ (
+ "no_params_returns_none_none",
+ 'await page.click(selector="button")',
+ None,
+ None,
+ ),
+ (
+ "no_params_returns_none_empty",
+ 'await page.click(selector="button")',
+ {},
+ None,
+ ),
+ (
+ "short_values_ignored",
+ """
async def block_fn(page, context):
if 'yes' == 'yes':
return {"next_block_label": "send_email", "branch_index": 0}
-"""
- params = {"flag": "yes", "ok": "true"}
- assert self.reviewer._validate_no_hardcoded_values(code, params) is None
-
- def test_detects_hardcoded_email(self) -> None:
- """Should catch a hardcoded email that should be context.parameters['recipient']."""
- code = """
-async def block_fn(page, context):
- recipient = '[email protected]'
- if recipient != "":
- return {"next_block_label": "send_email", "branch_index": 0}
-"""
- params = {"recipient": "[email protected]"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "hardcoded" in error.lower()
- assert "recipient" in error
- assert "context.parameters" in error # nosemgrep: incomplete-url-substring-sanitization
-
- def test_detects_hardcoded_url(self) -> None:
- """Should catch a hardcoded URL that should be context.parameters['website_url']."""
- code = """
-async def login(page, context):
- await page.goto('https://portal.vendor.com/login')
- await page.fill(selector='input[name="email"]', value=context.parameters['email'])
-"""
- params = {"website_url": "https://portal.vendor.com/login", "email": "[email protected]"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "website_url" in error
-
- def test_allows_context_parameters_reference(self) -> None:
- """Using context.parameters['key'] should NOT trigger the validator."""
- code = """
+""",
+ {"flag": "yes", "ok": "true"},
+ None,
+ ),
+ (
+ "allows_context_parameters_reference",
+ """
async def block_fn(page, context):
recipient = context.parameters.get('recipient', '')
if recipient != "":
return {"next_block_label": "send_email", "branch_index": 0}
-"""
- params = {"recipient": "[email protected]"}
- assert self.reviewer._validate_no_hardcoded_values(code, params) is None
-
- def test_allows_non_parameter_literals(self) -> None:
- """String literals that aren't parameter values should be fine."""
- code = """
+""",
+ {"recipient": "[email protected]"},
+ None,
+ ),
+ (
+ "allows_non_parameter_literals",
+ """
async def block_fn(page, context):
await page.click(selector='button:has-text("Sign in")', ai='fallback', prompt='Click sign in')
-"""
- params = {"email": "[email protected]", "password": "secret123"}
- assert self.reviewer._validate_no_hardcoded_values(code, params) is None
-
- def test_ignores_comments(self) -> None:
- """Values in comments should not trigger."""
- code = """
+""",
+ {"email": "[email protected]", "password": "secret123"},
+ None,
+ ),
+ (
+ "ignores_comments",
+ """
async def block_fn(page, context):
# [email protected] is the test email
recipient = context.parameters.get('recipient', '')
if recipient != "":
return {"next_block_label": "send_email", "branch_index": 0}
-"""
- params = {"recipient": "[email protected]"}
- assert self.reviewer._validate_no_hardcoded_values(code, params) is None
-
- def test_detects_multiple_hardcoded_values(self) -> None:
- """Should report multiple hardcoded values."""
- code = """
+""",
+ {"recipient": "[email protected]"},
+ None,
+ ),
+ (
+ "allows_block_labels_and_keywords",
+ """
async def block_fn(page, context):
+ return {"next_block_label": "send_email", "branch_index": 0}
+""",
+ {"account_number": "12345678"},
+ None,
+ ),
+ ],
+ id="allowed-and-early-exit",
+ ),
+ pytest.param(
+ [
+ (
+ "detects_hardcoded_email",
+ """
+async def block_fn(page, context):
+ recipient = '[email protected]'
+ if recipient != "":
+ return {"next_block_label": "send_email", "branch_index": 0}
+""",
+ {"recipient": "[email protected]"},
+ ("hardcoded", "recipient", "context.parameters"),
+ ),
+ (
+ "detects_hardcoded_url",
+ """
+async def login(page, context):
await page.goto('https://portal.vendor.com/login')
- await page.fill(selector='input', value='[email protected]')
-"""
- params = {
- "website_url": "https://portal.vendor.com/login",
- "recipient": "[email protected]",
- }
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "2 hardcoded" in error
-
- def test_allows_block_labels_and_keywords(self) -> None:
- """Common structural strings like block labels should not trigger."""
- code = """
+ await page.fill(selector='input[name="email"]', value=context.parameters['email'])
+""",
+ {"website_url": "https://portal.vendor.com/login", "email": "[email protected]"},
+ ("website_url",),
+ ),
+ (
+ "detects_multiple_hardcoded_values",
+ """
async def block_fn(page, context):
- return {"next_block_label": "send_email", "branch_index": 0}
-"""
- # Even if "send_email" is technically a value somewhere, it's a block label
- params = {"account_number": "12345678"}
- assert self.reviewer._validate_no_hardcoded_values(code, params) is None
-
- # ------------------------------------------------------------------
- # SKY-9295: substring matching — param values often appear embedded
- # inside longer click-prompt literals like
- # prompt='Should I download invoice 12/8/2025 for account 51410020?'
- # rather than as standalone literals. Pre-fix the matcher used exact
- # set membership (`param_value in code_literals`) and missed these.
- # ------------------------------------------------------------------
-
- def test_detects_param_embedded_in_click_prompt(self) -> None:
- """Account number embedded inside a click prompt string should trigger."""
- code = """
+ await page.goto('https://portal.vendor.com/login')
+ await page.fill(selector='input', value='[email protected]')
+""",
+ {
+ "website_url": "https://portal.vendor.com/login",
+ "recipient": "[email protected]",
+ },
+ ("2 hardcoded",),
+ ),
+ ],
+ id="exact-literal-detections",
+ ),
+ pytest.param(
+ [
+ (
+ "detects_param_embedded_in_click_prompt",
+ """
async def download_invoice(page, context):
await page.click(
ai='fallback',
prompt='Should I download the invoice dated 12/8/2025 for account 51410020?',
)
-"""
- params = {"account_number": "51410020", "download_start_date": "12/8/2025"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "account_number" in error
- assert "download_start_date" in error
-
- def test_detects_single_param_embedded_in_prompt(self) -> None:
- """Validator catches a substring even when only one param value is embedded."""
- code = """
+""",
+ {"account_number": "51410020", "download_start_date": "12/8/2025"},
+ ("account_number", "download_start_date"),
+ ),
+ (
+ "detects_single_param_embedded_in_prompt",
+ """
async def block_fn(page, context):
await page.click(prompt='Find the row for vendor CITYO49057A and click it')
-"""
- params = {"vendor_code": "CITYO49057A"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "vendor_code" in error
-
- def test_substring_match_respects_short_value_guard(self) -> None:
- """Short param values still don't false-positive even with substring matching."""
- # "yes" appears inside "yesterday" but is below the 5-char length guard.
- code = """
+""",
+ {"vendor_code": "CITYO49057A"},
+ ("vendor_code",),
+ ),
+ (
+ "substring_match_respects_short_value_guard",
+ """
async def block_fn(page, context):
await page.click(prompt='Click the yesterday button')
-"""
- params = {"flag": "yes"}
- assert self.reviewer._validate_no_hardcoded_values(code, params) is None
-
- def test_substring_match_skips_comments(self) -> None:
- """Embedded matches inside comments should still be ignored."""
- code = """
+""",
+ {"flag": "yes"},
+ None,
+ ),
+ (
+ "substring_match_skips_comments",
+ """
async def block_fn(page, context):
# Note: 51410020 is the test account number for staging
await page.click(prompt='Click the next button')
-"""
- params = {"account_number": "51410020"}
- assert self.reviewer._validate_no_hardcoded_values(code, params) is None
-
- def test_substring_match_skips_structural_literals(self) -> None:
- """Substring matching only applies to prose literals (whitespace + ≥20 chars).
- A short structural token like ``'next_block_label'`` should not trigger a
- false positive when a param value happens to be embedded in the token."""
- code = """
+""",
+ {"account_number": "51410020"},
+ None,
+ ),
+ (
+ "substring_match_skips_structural_literals",
+ """
async def block_fn(page, context):
return {"next_block_label": "send_email", "branch_index": 0}
-"""
- # "block" (5 chars) is a substring of "next_block_label" but
- # "next_block_label" is a structural keyword, not a prose literal — it
- # has no whitespace and is only 16 chars. The validator must ignore it.
- params = {"target": "block"}
- assert self.reviewer._validate_no_hardcoded_values(code, params) is None
-
- def test_validator_threshold_matches_generator_threshold(self) -> None:
- """Validator's minimum length must match the generator's substitution
- threshold (``MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB``). A 4-char value
- embedded in a prose literal is hardcoded and should be flagged."""
- from skyvern.core.script_generations.generate_script import MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB
-
- # Sanity: keep this test honest if the generator threshold ever changes.
- assert MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB == 4
-
- # Embed "1234" in a prose literal (whitespace + > prose-min-len).
- code = """
+""",
+ {"target": "block"},
+ None,
+ ),
+ ],
+ id="substring-matching",
+ ),
+ pytest.param(
+ [
+ (
+ "validator_threshold_matches_generator_threshold",
+ """
async def block_fn(page, context):
await page.click(prompt='Click the box labeled 1234 to continue')
-"""
- params = {"order_code": "1234"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "order_code" in error
-
- # ------------------------------------------------------------------
- # SKY-9295 v2: libcst-based literal extraction so triple-quoted f-strings
- # and other multiline prompts are no longer a blind spot.
- # ------------------------------------------------------------------
-
- def test_validator_catches_hardcoded_in_triple_quoted_prompt(self) -> None:
- """Generator emits triple-quoted f-strings when prompt content has
- newlines or quotes (generate_script.py:547-548). The pre-libcst regex
- scanner missed those. With libcst, literals from triple-quoted f-strings
- are visible to substring matching."""
- code = '''
+""",
+ {"order_code": "1234"},
+ ("order_code",),
+ ),
+ (
+ "validator_skips_oversized_param_values",
+ f'''
+async def block_fn(page, context):
+ note = "{"x" * (MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB + 1)}"
+''',
+ {"huge_payload": "x" * (MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB + 1)},
+ None,
+ ),
+ ],
+ id="generator-thresholds",
+ ),
+ pytest.param(
+ [
+ (
+ "validator_catches_hardcoded_in_triple_quoted_prompt",
+ '''
async def block_fn(page, context):
await page.click(
ai='fallback',
prompt=f"""Find the row for account 51410020
and click the download button""",
)
-'''
- params = {"account_number": "51410020"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "account_number" in error
-
- def test_validator_catches_hardcoded_in_fstring_text_segment(self) -> None:
- """Static text segments of an f-string (between {expr} interpolations)
- are scannable; if a parameter value is baked into the text portion, the
- validator must flag it."""
- code = """
+''',
+ {"account_number": "51410020"},
+ ("account_number",),
+ ),
+ (
+ "validator_catches_hardcoded_in_fstring_text_segment",
+ """
async def block_fn(page, context):
name = context.parameters['name']
await page.click(prompt=f'Send invoice 51410020 to {name}')
-"""
- params = {"account_number": "51410020"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "account_number" in error
-
- def test_validator_skips_fstring_interpolation_expression(self) -> None:
- """Interpolation expressions reference runtime values
- (``context.parameters[...]``), not literals — they shouldn't be scanned
- for hardcoded values. A reference to a key by string subscript inside
- an interpolation must NOT be flagged."""
- code = """
+""",
+ {"account_number": "51410020"},
+ ("account_number",),
+ ),
+ (
+ "validator_skips_fstring_interpolation_expression",
+ """
async def block_fn(page, context):
await page.click(prompt=f"Send to {context.parameters['recipient']}")
-"""
- params = {"recipient": "[email protected]"}
- # No literal contains "[email protected]"; only the interpolation
- # references the key. Validator must not false-positive.
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is None
-
- # ------------------------------------------------------------------
- # SKY-9295 v2: prose-literal threshold (whitespace + ≥ PROSE_LITERAL_MIN_LEN
- # chars). The floor is 8 (down from a stricter 20) so short embedded
- # prompts like ``'Click 51410020 now'`` are still caught.
- # ------------------------------------------------------------------
-
- def test_prose_literal_band_catches_embedded_value(self) -> None:
- """A short prose literal with whitespace catches an embedded param
- value via substring matching."""
- code = """
+""",
+ {"recipient": "[email protected]"},
+ None,
+ ),
+ ],
+ id="fstring-literals",
+ ),
+ pytest.param(
+ [
+ (
+ "prose_literal_band_catches_embedded_value",
+ """
async def block_fn(page, context):
await page.click(prompt='Click 51410020 now')
-"""
- # "Click 51410020 now" = 18 chars, has whitespace, ≥ 8. Prose-eligible.
- params = {"account_number": "51410020"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "account_number" in error
-
- def test_prose_literal_band_below_floor_skips(self) -> None:
- """Whitespace-bearing literals shorter than the prose floor are not
- eligible for substring matching (and exact-match still fails because
- the literal isn't equal to the param value)."""
- # "ok 1234" = 7 chars, < 8 → not prose; not equal to "1234" → not exact-match.
- code = """
+""",
+ {"account_number": "51410020"},
+ ("account_number",),
+ ),
+ (
+ "prose_literal_band_below_floor_skips",
+ """
async def block_fn(page, context):
await page.click(prompt='ok 1234')
-"""
- params = {"order_code": "1234"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is None
-
- def test_prose_literal_no_whitespace_skips(self) -> None:
- """Mid-length literals without whitespace are not prose — they're
- structural identifiers (``order_12345_id``, dict keys, etc.)."""
- code = """
+""",
+ {"order_code": "1234"},
+ None,
+ ),
+ (
+ "prose_literal_no_whitespace_skips",
+ """
async def block_fn(page, context):
return {"order_12345_id": "value", "branch_index": 0}
-"""
- params = {"order_id": "12345"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is None
-
- def test_prose_literal_band_exact_match_still_fires(self) -> None:
- """A param value that equals an entire short literal is still flagged
- via the exact-match path (which runs against every literal regardless
- of prose status)."""
- code = """
+""",
+ {"order_id": "12345"},
+ None,
+ ),
+ (
+ "prose_literal_band_exact_match_still_fires",
+ """
async def block_fn(page, context):
await page.fill(selector="#code", value="12345")
-"""
- params = {"order_code": "12345"}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is not None
- assert "order_code" in error
-
- # ------------------------------------------------------------------
- # SKY-9295 v2: MAX cap aligned with generator.
- # ------------------------------------------------------------------
-
- def test_validator_skips_oversized_param_values(self) -> None:
- """Param values longer than ``MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB``
- are skipped — the generator wouldn't have substituted them, so the
- validator must not flag them either (else generator/validator drift
- creates an unfixable diagnostic loop)."""
- from skyvern.core.script_generations.generate_script import MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB
-
- oversized = "x" * (MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB + 1)
- # Even though the literal contains the oversized value, it must be skipped.
- code = f'''
-async def block_fn(page, context):
- note = "{oversized}"
-'''
- params = {"huge_payload": oversized}
- error = self.reviewer._validate_no_hardcoded_values(code, params)
- assert error is None
+""",
+ {"order_code": "12345"},
+ ("order_code",),
+ ),
+ ],
+ id="prose-literal-band",
+ ),
+ ],
+ )
+ def test_no_hardcoded_value_cases(
+ self,
+ cases: list[tuple[str, str, dict[str, str] | None, tuple[str, ...] | None]],
+ ) -> None:
+ assert MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB == 4
+ for case_id, code, params, expected_substrings in cases:
+ error = self.reviewer._validate_no_hardcoded_values(code, params)
+ if expected_substrings is None:
+ assert error is None, case_id
+ continue
+ assert error is not None, case_id
+ for expected in expected_substrings:
+ assert expected in error, case_id
class TestValidateParameterPreservation:
@@ -339,78 +311,115 @@ class TestValidateParameterPreservation:
def setup_method(self) -> None:
self.reviewer = ScriptReviewer()
- def test_no_existing_code_returns_none(self) -> None:
- new_code = "await page.fill(selector='#email', ai='proactive', prompt='email')"
- assert self.reviewer._validate_parameter_preservation(new_code, None, ["email"]) is None
-
- def test_no_parameter_keys_returns_none(self) -> None:
- old_code = "await page.fill(selector='#email', value=context.parameters['email'])"
- new_code = "await page.fill(selector='#email', ai='proactive', prompt='email')"
- assert self.reviewer._validate_parameter_preservation(new_code, old_code, []) is None
-
- def test_preserved_refs_returns_none(self) -> None:
- """When all parameter refs are preserved, validation passes."""
- old_code = """
-await page.fill(selector='#email', value=context.parameters['email'])
-await page.fill(selector='#pass', value=context.parameters['password'])
-"""
- new_code = """
+ @pytest.mark.parametrize(
+ ("cases",),
+ [
+ pytest.param(
+ [
+ (
+ "no_existing_code_returns_none",
+ "await page.fill(selector='#email', ai='proactive', prompt='email')",
+ None,
+ ["email"],
+ None,
+ ),
+ (
+ "no_parameter_keys_returns_none",
+ "await page.fill(selector='#email', ai='proactive', prompt='email')",
+ "await page.fill(selector='#email', value=context.parameters['email'])",
+ [],
+ None,
+ ),
+ (
+ "preserved_refs_returns_none",
+ """
choice = await page.classify(...)
if choice == 0:
await page.fill(selector='#email', value=context.parameters['email'])
await page.fill(selector='#pass', value=context.parameters['password'])
-"""
- assert self.reviewer._validate_parameter_preservation(new_code, old_code, ["email", "password"]) is None
-
- def test_detects_dropped_refs(self) -> None:
- """When the LLM drops value= refs, validation catches it."""
- old_code = """
+""",
+ """
await page.fill(selector='#email', value=context.parameters['email'])
await page.fill(selector='#pass', value=context.parameters['password'])
-"""
- new_code = """
+""",
+ ["email", "password"],
+ None,
+ ),
+ (
+ "ignores_refs_not_in_parameter_keys",
+ "await page.fill(selector='#x', ai='proactive', prompt='fill x')",
+ "await page.fill(selector='#x', value=context.parameters['invented_key'])",
+ ["email", "password"],
+ None,
+ ),
+ ],
+ id="preserved-or-unconstrained",
+ ),
+ pytest.param(
+ [
+ (
+ "detects_dropped_refs",
+ """
choice = await page.classify(...)
if choice == 0:
await page.fill(selector='#email', ai='proactive', prompt='fill email')
await page.fill(selector='#pass', ai='proactive', prompt='fill password')
-"""
- error = self.reviewer._validate_parameter_preservation(new_code, old_code, ["email", "password"])
- assert error is not None
- assert "email" in error
- assert "password" in error
- assert "dropped" in error.lower()
-
- def test_ignores_refs_not_in_parameter_keys(self) -> None:
- """Spurious refs in old code that aren't valid keys should be ignored."""
- old_code = "await page.fill(selector='#x', value=context.parameters['invented_key'])"
- new_code = "await page.fill(selector='#x', ai='proactive', prompt='fill x')"
- assert self.reviewer._validate_parameter_preservation(new_code, old_code, ["email", "password"]) is None
-
- def test_partial_drop_detected(self) -> None:
- """Dropping one ref while keeping another should flag only the dropped one."""
- old_code = """
+""",
+ """
await page.fill(selector='#email', value=context.parameters['email'])
await page.fill(selector='#pass', value=context.parameters['password'])
-"""
- new_code = """
-await page.fill(selector='#email', value=context.parameters['email'])
-await page.fill(selector='#pass', ai='proactive', prompt='fill password')
-"""
- error = self.reviewer._validate_parameter_preservation(new_code, old_code, ["email", "password"])
- assert error is not None
- assert "context.parameters['password']" in error
- assert "context.parameters['email']" not in error
-
- def test_commented_ref_in_new_code_counts_as_dropped(self) -> None:
- """A parameter ref only in a comment in new code should be flagged as dropped."""
- old_code = "await page.fill(selector='#email', value=context.parameters['email'])"
- new_code = """
+""",
+ ["email", "password"],
+ ("email", "password", "dropped"),
+ ),
+ (
+ "commented_ref_in_new_code_counts_as_dropped",
+ """
# was: context.parameters['email']
await page.fill(selector='#email', ai='proactive', prompt='fill email')
-"""
- error = self.reviewer._validate_parameter_preservation(new_code, old_code, ["email"])
- assert error is not None
- assert "email" in error
+""",
+ "await page.fill(selector='#email', value=context.parameters['email'])",
+ ["email"],
+ ("email",),
+ ),
+ ],
+ id="dropped-references",
+ ),
+ pytest.param(
+ [
+ (
+ "partial_drop_detected",
+ """
+await page.fill(selector='#email', value=context.parameters['email'])
+await page.fill(selector='#pass', ai='proactive', prompt='fill password')
+""",
+ """
+await page.fill(selector='#email', value=context.parameters['email'])
+await page.fill(selector='#pass', value=context.parameters['password'])
+""",
+ ["email", "password"],
+ ("context.parameters['password']", "!context.parameters['email']"),
+ )
+ ],
+ id="partial-drop",
+ ),
+ ],
+ )
+ def test_parameter_preservation_cases(
+ self,
+ cases: list[tuple[str, str, str | None, list[str], tuple[str, ...] | None]],
+ ) -> None:
+ for case_id, new_code, old_code, parameter_keys, expected_substrings in cases:
+ error = self.reviewer._validate_parameter_preservation(new_code, old_code, parameter_keys)
+ if expected_substrings is None:
+ assert error is None, case_id
+ continue
+ assert error is not None, case_id
+ for expected in expected_substrings:
+ if expected.startswith("!"):
+ assert expected[1:] not in error, case_id
+ else:
+ assert expected.lower() in error.lower(), case_id
class TestValidateBranchReturns:
@@ -421,126 +430,167 @@ class TestValidateBranchReturns:
{"original_expression": None, "next_block_label": "block_4", "is_default": True},
]
- def test_valid_returns_passes(self) -> None:
- code = """
+ @pytest.mark.parametrize(
+ ("cases",),
+ [
+ pytest.param(
+ [
+ (
+ "valid_returns_passes",
+ """
async def block_fn(page, context):
if context.parameters.get('x', 0) > 0:
return {"next_block_label": "block_3", "branch_index": 0}
else:
return {"next_block_label": "block_4", "branch_index": 1}
-"""
- assert ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO) is None
-
- def test_invalid_label_detected(self) -> None:
- """Labels not in the branch definitions should be flagged."""
- code = """
+""",
+ BRANCHES_TWO,
+ None,
+ ),
+ (
+ "none_label_valid_when_in_branches",
+ """
+async def block_fn(page, context):
+ if context.parameters.get('x', 0) > 0:
+ return {"next_block_label": "block_3", "branch_index": 0}
+ else:
+ return {"next_block_label": None, "branch_index": 1}
+""",
+ [
+ {"original_expression": "x > 0", "next_block_label": "block_3", "is_default": False},
+ {"original_expression": None, "next_block_label": None, "is_default": True},
+ ],
+ None,
+ ),
+ (
+ "single_quotes_handled",
+ """
+async def block_fn(page, context):
+ return {'next_block_label': 'block_3', 'branch_index': 0}
+""",
+ BRANCHES_TWO,
+ None,
+ ),
+ ],
+ id="valid-literals",
+ ),
+ pytest.param(
+ [
+ (
+ "invalid_label_detected",
+ """
async def block_fn(page, context):
if context.parameters.get('x', 0) > 0:
return {"next_block_label": "block_99", "branch_index": 0}
else:
return {"next_block_label": "block_4", "branch_index": 1}
-"""
- error = ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO)
- assert error is not None
- assert "block_99" in error
-
- def test_invalid_index_detected(self) -> None:
- """branch_index values outside 0..N-1 should be flagged."""
- code = """
+""",
+ BRANCHES_TWO,
+ ("block_99",),
+ ),
+ (
+ "invalid_index_detected",
+ """
async def block_fn(page, context):
if context.parameters.get('x', 0) > 0:
return {"next_block_label": "block_3", "branch_index": 0}
else:
return {"next_block_label": "block_4", "branch_index": -1}
-"""
- error = ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO)
- assert error is not None
- assert "-1" in error
-
- def test_none_label_invalid_when_not_in_branches(self) -> None:
- """None next_block_label should be flagged when no branch has a null target."""
- code = """
+""",
+ BRANCHES_TWO,
+ ("-1",),
+ ),
+ (
+ "none_label_invalid_when_not_in_branches",
+ """
async def block_fn(page, context):
if context.parameters.get('x', 0) > 0:
return {"next_block_label": "block_3", "branch_index": 0}
else:
return {"next_block_label": None, "branch_index": 1}
-"""
- error = ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO)
- assert error is not None
- assert "next_block_label" in error
-
- def test_none_label_valid_when_in_branches(self) -> None:
- """None next_block_label should pass when a branch has a null target."""
- branches = [
- {"original_expression": "x > 0", "next_block_label": "block_3", "is_default": False},
- {"original_expression": None, "next_block_label": None, "is_default": True},
- ]
- code = """
+""",
+ BRANCHES_TWO,
+ ("next_block_label",),
+ ),
+ (
+ "both_label_and_index_invalid",
+ """
async def block_fn(page, context):
- if context.parameters.get('x', 0) > 0:
- return {"next_block_label": "block_3", "branch_index": 0}
- else:
- return {"next_block_label": None, "branch_index": 1}
-"""
- assert ScriptReviewer._validate_branch_returns(code, branches) is None
-
- def test_no_literals_passes(self) -> None:
- """Code using variables (not literals) should pass — can't validate statically."""
- code = """
+ return {"next_block_label": "wrong_label", "branch_index": -1}
+""",
+ BRANCHES_TWO,
+ ("wrong_label", "-1"),
+ ),
+ ],
+ id="invalid-literals",
+ ),
+ pytest.param(
+ [
+ (
+ "no_literals_passes",
+ """
async def block_fn(page, context):
label = compute_label()
idx = compute_index()
return {"next_block_label": label, "branch_index": idx}
-"""
- assert ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO) is None
-
- def test_comments_ignored(self) -> None:
- """Values in comments should not trigger validation."""
- code = """
+""",
+ BRANCHES_TWO,
+ None,
+ ),
+ (
+ "comments_ignored",
+ """
async def block_fn(page, context):
# return {"next_block_label": "block_99", "branch_index": -1}
if context.parameters.get('x', 0) > 0:
return {"next_block_label": "block_3", "branch_index": 0}
else:
return {"next_block_label": "block_4", "branch_index": 1}
-"""
- assert ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO) is None
-
- def test_empty_branches_passes(self) -> None:
- assert ScriptReviewer._validate_branch_returns("return {}", []) is None
-
- def test_single_quotes_handled(self) -> None:
- """Single-quoted keys should be parsed correctly."""
- code = """
-async def block_fn(page, context):
- return {'next_block_label': 'block_3', 'branch_index': 0}
-"""
- assert ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO) is None
-
- def test_both_label_and_index_invalid(self) -> None:
- """Both invalid label and index should be reported."""
- code = """
-async def block_fn(page, context):
- return {"next_block_label": "wrong_label", "branch_index": -1}
-"""
- error = ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO)
- assert error is not None
- assert "wrong_label" in error
- assert "-1" in error
+""",
+ BRANCHES_TWO,
+ None,
+ ),
+ ],
+ id="ignored-static-unvalidated",
+ ),
+ pytest.param(
+ [
+ (
+ "empty_branches_passes",
+ "return {}",
+ [],
+ None,
+ )
+ ],
+ id="empty-branches",
+ ),
+ ],
+ )
+ def test_branch_return_cases(
+ self,
+ cases: list[tuple[str, str, list[dict], tuple[str, ...] | None]],
+ ) -> None:
+ for case_id, code, branches, expected_substrings in cases:
+ error = ScriptReviewer._validate_branch_returns(code, branches)
+ if expected_substrings is None:
+ assert error is None, case_id
+ continue
+ assert error is not None, case_id
+ for expected in expected_substrings:
+ assert expected in error, case_id
class TestLoadFilteredRunParamValues:
- """Tests for load_filtered_run_param_values (shared loader for 3 reviewer ingestion sites)."""
+ """Tests for load_filtered_run_param_values."""
- def _make_workflow_param(self, key: str) -> object:
- from datetime import datetime
-
- from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType
+ def _make_workflow_param(
+ self, key: str, parameter_type: WorkflowParameterType = WorkflowParameterType.STRING
+ ) -> object:
+ from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter
return WorkflowParameter(
workflow_parameter_id=f"wp_{key}",
- workflow_parameter_type=WorkflowParameterType.STRING,
+ workflow_parameter_type=parameter_type,
workflow_id="w_1",
key=key,
created_at=datetime(2026, 1, 1),
@@ -548,8 +598,6 @@ class TestLoadFilteredRunParamValues:
)
def _make_credential_param(self, key: str) -> object:
- from datetime import datetime
-
from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter
return CredentialParameter(
@@ -561,129 +609,127 @@ class TestLoadFilteredRunParamValues:
modified_at=datetime(2026, 1, 1),
)
- async def _run_with_params(self, monkeypatch: object, param_tuples: list) -> dict[str, str]:
+ async def _run_with_params(self, monkeypatch: pytest.MonkeyPatch, param_tuples: list) -> dict[str, str]:
from skyvern.services.script_reviewer import load_filtered_run_param_values
async def fake_get(workflow_run_id: str) -> list:
return param_tuples
- # Patch DATABASE.workflow_runs.get_workflow_run_parameters
- from skyvern.forge import app
-
- monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run_parameters", fake_get) # type: ignore[attr-defined]
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run_parameters", fake_get)
return await load_filtered_run_param_values("wr_test")
+ @pytest.mark.parametrize(
+ ("case_id", "param_tuples", "expected"),
+ [
+ pytest.param(
+ "filters_sensitive_credential_param",
+ "credential-param",
+ {"account_number": "51410020"},
+ id="credential-param-filtered",
+ ),
+ pytest.param(
+ "filters_credential_id_subtype",
+ "credential-id-subtype",
+ {"account_number": "51410020"},
+ id="credential-id-filtered",
+ ),
+ pytest.param("skips_blank_values", "blank-values", {"real": "51410020"}, id="blank-values-skipped"),
+ ],
+ )
@pytest.mark.asyncio
- async def test_filters_sensitive_credential_param(self, monkeypatch: object) -> None:
- from types import SimpleNamespace
-
- wf_param_secret = self._make_credential_param("login")
- wf_param_normal = self._make_workflow_param("account_number")
- param_tuples = [
- (wf_param_secret, SimpleNamespace(value="cred_xyz")),
- (wf_param_normal, SimpleNamespace(value="51410020")),
- ]
- result = await self._run_with_params(monkeypatch, param_tuples)
- assert result == {"account_number": "51410020"}
-
- @pytest.mark.asyncio
- async def test_filters_credential_id_subtype(self, monkeypatch: object) -> None:
- from datetime import datetime
- from types import SimpleNamespace
-
- from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType
-
- wf_param_credential_id = WorkflowParameter(
- workflow_parameter_id="wp_cred",
- workflow_parameter_type=WorkflowParameterType.CREDENTIAL_ID,
- workflow_id="w_1",
- key="selected_credential",
- created_at=datetime(2026, 1, 1),
- modified_at=datetime(2026, 1, 1),
- )
- wf_param_normal = self._make_workflow_param("account_number")
- param_tuples = [
- (wf_param_credential_id, SimpleNamespace(value="cred_abc")),
- (wf_param_normal, SimpleNamespace(value="51410020")),
- ]
- result = await self._run_with_params(monkeypatch, param_tuples)
- assert "selected_credential" not in result
- assert result == {"account_number": "51410020"}
+ async def test_load_filtered_run_param_value_cases(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ case_id: str,
+ param_tuples: str,
+ expected: dict[str, str],
+ ) -> None:
+ if param_tuples == "credential-param":
+ tuples = [
+ (self._make_credential_param("login"), SimpleNamespace(value="cred_xyz")),
+ (self._make_workflow_param("account_number"), SimpleNamespace(value="51410020")),
+ ]
+ elif param_tuples == "credential-id-subtype":
+ tuples = [
+ (
+ self._make_workflow_param("selected_credential", WorkflowParameterType.CREDENTIAL_ID),
+ SimpleNamespace(value="cred_abc"),
+ ),
+ (self._make_workflow_param("account_number"), SimpleNamespace(value="51410020")),
+ ]
+ else:
+ tuples = [
+ (self._make_workflow_param("blank"), SimpleNamespace(value=" ")),
+ (self._make_workflow_param("none_val"), SimpleNamespace(value=None)),
+ (self._make_workflow_param("real"), SimpleNamespace(value="51410020")),
+ ]
+ result = await self._run_with_params(monkeypatch, tuples)
+ assert result == expected, case_id
@pytest.mark.asyncio
- async def test_db_failure_returns_empty(self, monkeypatch: object) -> None:
- from skyvern.forge import app
+ async def test_db_failure_returns_empty(self, monkeypatch: pytest.MonkeyPatch) -> None:
from skyvern.services.script_reviewer import load_filtered_run_param_values
async def fake_get_raises(workflow_run_id: str) -> list:
raise RuntimeError("DB connection lost")
- monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run_parameters", fake_get_raises) # type: ignore[attr-defined]
+ monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run_parameters", fake_get_raises)
result = await load_filtered_run_param_values("wr_test")
assert result == {}
- @pytest.mark.asyncio
- async def test_skips_blank_values(self, monkeypatch: object) -> None:
- from types import SimpleNamespace
-
- wf_param_a = self._make_workflow_param("blank")
- wf_param_b = self._make_workflow_param("none_val")
- wf_param_c = self._make_workflow_param("real")
- param_tuples = [
- (wf_param_a, SimpleNamespace(value=" ")), # whitespace-only
- (wf_param_b, SimpleNamespace(value=None)),
- (wf_param_c, SimpleNamespace(value="51410020")),
- ]
- result = await self._run_with_params(monkeypatch, param_tuples)
- assert result == {"real": "51410020"}
-
class TestFilterSelfOutputParamKeys:
- """Tests for _filter_self_output_param_keys.
-
- Each block carries an auto-generated ``<block_label>_output`` parameter.
- Inside the block's own cached function that slot is ``None`` until the
- block has finished running, so any ``context.parameters['<self>_output']``
- reference crashes with ``'NoneType' object is not subscriptable``.
- Upstream and downstream block outputs are valid references — only the
- *current* block's output must be filtered.
- """
-
- def test_drops_self_output_only(self) -> None:
- keys = ["account_number", "Login_block_output", "Search_block_output"]
- filtered = ScriptReviewer._filter_self_output_param_keys(keys, "Login_block")
- assert filtered == ["account_number", "Search_block_output"]
-
- def test_keeps_upstream_outputs(self) -> None:
- """Upstream `_output` keys are legitimate cross-block references and must survive."""
- keys = ["Search_results_output", "Login_block_output"]
- filtered = ScriptReviewer._filter_self_output_param_keys(keys, "Login_block")
- assert "Search_results_output" in filtered
- assert "Login_block_output" not in filtered
-
- def test_keeps_downstream_outputs(self) -> None:
- """Downstream `_output` keys aren't read by the current block in practice,
- but filtering only the self-output keeps the rule narrow and predictable."""
- keys = ["Login_block_output", "Final_extract_output"]
- filtered = ScriptReviewer._filter_self_output_param_keys(keys, "Login_block")
- assert "Final_extract_output" in filtered
- assert "Login_block_output" not in filtered
-
- def test_empty_block_label_is_noop(self) -> None:
- """An empty block_label means we don't know what to filter — pass through."""
- keys = ["a", "b_output"]
- assert ScriptReviewer._filter_self_output_param_keys(keys, "") == keys
-
- def test_no_self_output_present(self) -> None:
- keys = ["account_number", "username"]
- assert ScriptReviewer._filter_self_output_param_keys(keys, "Login_block") == keys
-
- def test_partial_match_not_filtered(self) -> None:
- """`block_output_other` matches `<block>_output` as a prefix but isn't the
- self-output key. The filter must be exact-match, not prefix-match."""
- keys = ["Login_block_output_other", "Login_block_output"]
- filtered = ScriptReviewer._filter_self_output_param_keys(keys, "Login_block")
- assert filtered == ["Login_block_output_other"]
+ """Tests for _filter_self_output_param_keys."""
+
+ @pytest.mark.parametrize(
+ ("cases",),
+ [
+ pytest.param(
+ [
+ (
+ "drops_self_output_only",
+ ["account_number", "Login_block_output", "Search_block_output"],
+ "Login_block",
+ ["account_number", "Search_block_output"],
+ ),
+ (
+ "keeps_upstream_outputs",
+ ["Search_results_output", "Login_block_output"],
+ "Login_block",
+ ["Search_results_output"],
+ ),
+ (
+ "keeps_downstream_outputs",
+ ["Login_block_output", "Final_extract_output"],
+ "Login_block",
+ ["Final_extract_output"],
+ ),
+ (
+ "partial_match_not_filtered",
+ ["Login_block_output_other", "Login_block_output"],
+ "Login_block",
+ ["Login_block_output_other"],
+ ),
+ ],
+ id="self-output-filtered-exactly",
+ ),
+ pytest.param(
+ [
+ ("empty_block_label_is_noop", ["a", "b_output"], "", ["a", "b_output"]),
+ (
+ "no_self_output_present",
+ ["account_number", "username"],
+ "Login_block",
+ ["account_number", "username"],
+ ),
+ ],
+ id="no-filter-needed",
+ ),
+ ],
+ )
+ def test_filter_self_output_param_key_cases(self, cases: list[tuple[str, list[str], str, list[str]]]) -> None:
+ for case_id, keys, block_label, expected in cases:
+ assert ScriptReviewer._filter_self_output_param_keys(keys, block_label) == expected, case_id
class TestValidateParameterReferencesSelfOutput:
@@ -692,246 +738,184 @@ class TestValidateParameterReferencesSelfOutput:
def setup_method(self) -> None:
self.reviewer = ScriptReviewer()
- def test_self_output_reference_rejected(self) -> None:
- """When `parameter_keys` excludes the self-output, a `context.parameters['<self>_output']`
- reference in the LLM output must be rejected by the validator."""
- code = """
+ @pytest.mark.parametrize(
+ ("cases",),
+ [
+ pytest.param(
+ [
+ (
+ "self_output_reference_rejected",
+ """
async def block_fn(page, context):
await page.fill(selector='#user', value=context.parameters['Login_block_output']['username'])
-"""
- # parameter_keys does NOT contain Login_block_output (already filtered)
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- def test_self_output_error_message_explains_why(self) -> None:
- """The retry hint should specifically explain the self-output crash so the LLM
- doesn't reintroduce the same pattern on the next attempt."""
- code = "value=context.parameters['Login_block_output']"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "None" in error # message should mention the None / NoneType crash mode
-
- def test_validator_unchanged_without_self_output_key(self) -> None:
- """Existing callers that don't pass `self_output_key` keep the prior behavior."""
- code = "value=context.parameters['fake_key']"
- error = self.reviewer._validate_parameter_references(code, parameter_keys=["real_key"])
- assert error is not None
- assert "fake_key" in error
- # No self-output augmentation when caller didn't opt in.
- assert "self-output" not in error.lower()
-
- # ------------------------------------------------------------------
- # Empty-valid-set edge case: when the workflow's only declared parameter
- # was the block's own output (filtered → empty parameter_keys), the
- # validator must still scan to catch a self-output ref.
- # ------------------------------------------------------------------
-
- def test_self_output_rejected_with_empty_parameter_keys(self) -> None:
- """When parameter_keys is empty BUT self_output_key is set, the
- validator must still reject self-output references — otherwise the bug
- slips through any workflow whose only parameter was the block's own
- output."""
- code = "value=context.parameters['Login_block_output']['username']"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=[],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- def test_no_constraints_returns_none(self) -> None:
- """No parameter_keys AND no self_output_key → no validation possible."""
- code = "value=context.parameters['anything']"
- assert self.reviewer._validate_parameter_references(code, parameter_keys=[]) is None
-
- def test_empty_valid_set_allows_unknown_non_self_refs(self) -> None:
- """Empty ``parameter_keys`` + ``self_output_key`` in scope: the reviewer
- rejects only the self-output ref. Other unknown refs may be runtime-
- valid synthesized keys (``GeneratedWorkflowParameters`` field names)
- that the reviewer doesn't load into its valid set, so we let them
- through rather than risking false-positive rejection of legitimate
- deterministic-named parameters.
- """
- code = "value=context.parameters['some_other_key']"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=[],
- self_output_key="Login_block_output",
- )
- assert error is None
-
- # ------------------------------------------------------------------
- # `.get('key')` access: regex must catch this form too — the reviewer
- # prompt's example output uses ``.get(...)``, so the LLM can emit it.
- # ------------------------------------------------------------------
-
- def test_get_access_self_output_rejected(self) -> None:
- """`context.parameters.get('<self>_output')` must be caught the same as
- the subscript form."""
- code = "value=context.parameters.get('Login_block_output')"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- def test_get_access_with_default_self_output_rejected(self) -> None:
- """The `.get(key, default)` two-argument form is also caught."""
- code = "value=context.parameters.get('Login_block_output', '')"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- def test_get_access_invented_key_rejected(self) -> None:
- """Generic invented-key detection also applies via the `.get()` regex."""
- code = "value=context.parameters.get('made_up_field')"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- )
- assert error is not None
- assert "made_up_field" in error
-
- # ------------------------------------------------------------------
- # Whitespace inside subscript brackets / `.get()` parens. Reformatted
- # variants like ``[ 'key' ]`` must not bypass the validator.
- # ------------------------------------------------------------------
-
- def test_spaced_subscript_self_output_rejected(self) -> None:
- """`context.parameters[ 'X_output' ]['username']` must be caught despite
- whitespace inside the outer subscript brackets."""
- code = "value=context.parameters[ 'Login_block_output' ]['username']"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- def test_spaced_get_access_self_output_rejected(self) -> None:
- """`.get( 'X_output' )` form with whitespace must also be caught."""
- code = "value=context.parameters.get( 'Login_block_output' )"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- # ------------------------------------------------------------------
- # Multiline access: ``ruff format`` can wrap long subscripts across
- # multiple lines. The validator must still match.
- # ------------------------------------------------------------------
-
- def test_multiline_subscript_self_output_rejected(self) -> None:
- """A subscript split across multiple lines must still be caught."""
- code = "value=context.parameters[\n 'Login_block_output'\n]['username']"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- def test_multiline_get_access_self_output_rejected(self) -> None:
- """A `.get(...)` call split across multiple lines must still be caught."""
- code = "value=context.parameters.get(\n 'Login_block_output',\n {},\n)"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- # ------------------------------------------------------------------
- # Whitespace between ``parameters`` and the accessor: valid Python
- # (``context.parameters [...]``, ``context.parameters .get(...)``)
- # must be matched too.
- # ------------------------------------------------------------------
-
- def test_space_before_subscript_self_output_rejected(self) -> None:
- """`context.parameters ['X_output']` (space between `parameters` and `[`)
- must be caught."""
- code = "value=context.parameters ['Login_block_output']"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- def test_space_before_get_self_output_rejected(self) -> None:
- """`context.parameters .get('X_output')` (space between `parameters` and
- `.`) must be caught."""
- code = "value=context.parameters .get('Login_block_output')"
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- self_output_key="Login_block_output",
- )
- assert error is not None
- assert "Login_block_output" in error
-
- def test_multiline_ref_inside_comment_block_not_flagged(self) -> None:
- """Multiline scan must still skip refs that live inside comment lines.
- Each comment line is replaced with a blank line before scanning, so a
- block of comment-prefixed lines containing a fake ref must not be
- flagged as a violation."""
- code = (
- "# context.parameters['fake_key']\n"
- "# context.parameters.get('another_fake_key')\n"
- "value=context.parameters['account_number']\n"
- )
- error = self.reviewer._validate_parameter_references(
- code,
- parameter_keys=["account_number"],
- )
- assert error is None
+""",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ (
+ "self_output_error_message_explains_why",
+ "value=context.parameters['Login_block_output']",
+ ["account_number"],
+ "Login_block_output",
+ ("None",),
+ ),
+ (
+ "self_output_rejected_with_empty_parameter_keys",
+ "value=context.parameters['Login_block_output']['username']",
+ [],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ ],
+ id="self-output-rejected",
+ ),
+ pytest.param(
+ [
+ (
+ "validator_unchanged_without_self_output_key",
+ "value=context.parameters['fake_key']",
+ ["real_key"],
+ None,
+ ("fake_key", "!self-output"),
+ ),
+ (
+ "get_access_invented_key_rejected",
+ "value=context.parameters.get('made_up_field')",
+ ["account_number"],
+ None,
+ ("made_up_field",),
+ ),
+ ],
+ id="invented-keys",
+ ),
+ pytest.param(
+ [
+ (
+ "no_constraints_returns_none",
+ "value=context.parameters['anything']",
+ [],
+ None,
+ None,
+ ),
+ (
+ "empty_valid_set_allows_unknown_non_self_refs",
+ "value=context.parameters['some_other_key']",
+ [],
+ "Login_block_output",
+ None,
+ ),
+ (
+ "multiline_ref_inside_comment_block_not_flagged",
+ (
+ "# context.parameters['fake_key']\n"
+ "# context.parameters.get('another_fake_key')\n"
+ "value=context.parameters['account_number']\n"
+ ),
+ ["account_number"],
+ None,
+ None,
+ ),
+ ],
+ id="allowed-reference-shapes",
+ ),
+ pytest.param(
+ [
+ (
+ "get_access_self_output_rejected",
+ "value=context.parameters.get('Login_block_output')",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ (
+ "get_access_with_default_self_output_rejected",
+ "value=context.parameters.get('Login_block_output', '')",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ (
+ "spaced_subscript_self_output_rejected",
+ "value=context.parameters[ 'Login_block_output' ]['username']",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ (
+ "spaced_get_access_self_output_rejected",
+ "value=context.parameters.get( 'Login_block_output' )",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ (
+ "multiline_subscript_self_output_rejected",
+ "value=context.parameters[\n 'Login_block_output'\n]['username']",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ (
+ "multiline_get_access_self_output_rejected",
+ "value=context.parameters.get(\n 'Login_block_output',\n {},\n)",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ (
+ "space_before_subscript_self_output_rejected",
+ "value=context.parameters ['Login_block_output']",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ (
+ "space_before_get_self_output_rejected",
+ "value=context.parameters .get('Login_block_output')",
+ ["account_number"],
+ "Login_block_output",
+ ("Login_block_output",),
+ ),
+ ],
+ id="self-output-access-variants",
+ ),
+ ],
+ )
+ def test_parameter_reference_cases(
+ self,
+ cases: list[tuple[str, str, list[str], str | None, tuple[str, ...] | None]],
+ ) -> None:
+ for case_id, code, parameter_keys, self_output_key, expected_substrings in cases:
+ error = self.reviewer._validate_parameter_references(
+ code,
+ parameter_keys=parameter_keys,
+ self_output_key=self_output_key,
+ )
+ if expected_substrings is None:
+ assert error is None, case_id
+ continue
+ assert error is not None, case_id
+ for expected in expected_substrings:
+ if expected.startswith("!"):
+ assert expected[1:] not in error.lower(), case_id
+ else:
+ assert expected in error, case_id
class TestConditionalCodeSelfOutputRejection:
- """The conditional-review path also persists code via
- ``create_script_version_from_review``, so it must run the same self-output
- parameter validation as ``_review_block_internal``."""
+ """The conditional-review path must reject self-output parameter references."""
def setup_method(self) -> None:
self.reviewer = ScriptReviewer()
@pytest.mark.asyncio
- async def test_generate_conditional_code_calls_validator_with_self_output(self, monkeypatch: object) -> None:
- """End-to-end wiring: drive ``_generate_conditional_code`` with a mocked
- LLM that returns code containing the self-output pattern, and assert
- ``_validate_parameter_references`` was invoked with the block's
- ``self_output_key``. A future refactor that drops the validator call
- or threads the wrong ``block_label`` will make this test fail."""
+ async def test_generate_conditional_code_calls_validator_with_self_output(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
start_forge_stub_app()
block_label = "Branch_block"
-
- # Conditional code containing the self-output crash pattern. The mocked
- # LLM returns this twice (max_attempts=2) so the validator's rejection
- # path runs both attempts.
bad_code = (
"async def block_fn(page, context):\n"
" if context.parameters['Branch_block_output']['flag']:\n"
@@ -945,7 +929,6 @@ class TestConditionalCodeSelfOutputRejection:
app.SCRIPT_REVIEWER_LLM_API_HANDLER = AsyncMock(side_effect=fake_llm_handler)
- # Spy on the validator to confirm the wiring threads `self_output_key`.
validator_calls: list[dict[str, object]] = []
original_validator = self.reviewer._validate_parameter_references
@@ -957,8 +940,6 @@ class TestConditionalCodeSelfOutputRejection:
monkeypatch.setattr(self.reviewer, "_validate_parameter_references", spy_validator)
- # Build a minimal episode with a branch expression so
- # `_generate_conditional_code` proceeds past its early returns.
episode = ScriptFallbackEpisode(
episode_id="ep_1",
organization_id="o_test",
@@ -996,25 +977,14 @@ class TestConditionalCodeSelfOutputRejection:
all_parameter_keys=["account_number", f"{block_label}_output"],
)
- # Validator must have been called at least once with the correct
- # self_output_key for this block. This is the wiring assertion.
assert validator_calls, "validator was never invoked — wiring broken"
assert all(call["self_output_key"] == f"{block_label}_output" for call in validator_calls), (
f"validator received wrong self_output_key: {validator_calls}"
)
-
- # The bad_code references the block's self-output, so the validator
- # rejects it on every attempt and the function returns None (no code
- # accepted for persistence). This confirms the rejection chain reaches
- # the persistence gate.
assert result is None, "self-output ref should have been rejected before persistence"
@pytest.mark.asyncio
- async def test_review_conditional_blocks_threads_parameter_keys(self, monkeypatch: object) -> None:
- """Outer-wiring test: ``review_conditional_blocks`` loads workflow
- context internally and threads ``all_parameter_keys`` into
- ``_generate_conditional_code``. A refactor that drops that load (or
- threads the wrong keys) must fail this test."""
+ async def test_review_conditional_blocks_threads_parameter_keys(self, monkeypatch: pytest.MonkeyPatch) -> None:
start_forge_stub_app()
block_label = "Branch_block"
@@ -1024,22 +994,18 @@ class TestConditionalCodeSelfOutputRejection:
app.SCRIPT_REVIEWER_LLM_API_HANDLER = AsyncMock(side_effect=fake_llm_handler)
- # Spy on `_load_workflow_context` to confirm it's called from the
- # conditional path AND to control what it returns.
load_calls: list[dict[str, str]] = []
async def fake_load(organization_id: str, workflow_permanent_id: str) -> tuple:
load_calls.append({"organization_id": organization_id, "workflow_permanent_id": workflow_permanent_id})
return (
- {block_label: ""}, # goals
- ["account_number", f"{block_label}_output"], # all_parameter_keys
- {}, # block_criteria
+ {block_label: ""},
+ ["account_number", f"{block_label}_output"],
+ {},
)
monkeypatch.setattr(self.reviewer, "_load_workflow_context", fake_load)
- # Spy on the validator to confirm `_generate_conditional_code` was
- # called WITH the keys threaded through `_load_workflow_context`.
validator_calls: list[dict[str, object]] = []
original_validator = self.reviewer._validate_parameter_references
@@ -1078,11 +1044,9 @@ class TestConditionalCodeSelfOutputRejection:
run_parameter_values=None,
)
- # Assertion 1: workflow context was loaded by review_conditional_blocks itself.
assert load_calls == [{"organization_id": "o_test", "workflow_permanent_id": "wpid_test"}], (
"review_conditional_blocks did not load workflow context — the wiring is broken"
)
- # Assertion 2: the validator received the loaded keys, with the self-output filtered out.
assert validator_calls, "validator was never invoked from the conditional path"
for call in validator_calls:
assert "account_number" in call["param_keys"], "upstream key should have been threaded through"
@@ -1090,8 +1054,6 @@ class TestConditionalCodeSelfOutputRejection:
assert call["self_output_key"] == f"{block_label}_output", "self_output_key not threaded correctly"
def test_validator_directly_rejects_self_output(self) -> None:
- """Direct unit-level assertion (kept alongside the integration test):
- validator returns an error message identifying the self-output key."""
block_label = "Branch_block"
code_with_self_output = (
"async def block_fn(page, context):\n"