Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
test(SKY-11847): case-table the script-reviewer validator tests (118→46, coverage-identical) (#7119)
Suchintan4dc6094473d1
diff --git a/tests/unit/test_script_reviewer_quality.py b/tests/unit/test_script_reviewer_quality.pyindex eadf8c468..e92cd3506 100644--- a/tests/unit/test_script_reviewer_quality.py+++ b/tests/unit/test_script_reviewer_quality.py@@ -2,6 +2,8 @@from pathlib import Path+import pytest+from skyvern.core.script_generations.script_validators import validate_missing_selectorsfrom skyvern.services.script_reviewer import ScriptReviewer@@ -12,17 +14,9 @@ class TestSelectorReplacementGuidance:"""Pin Rule 8b's multi-row form guidance against accidental deletion."""def test_rule_8b_covers_multi_row_full_block_case(self) -> None:- # Assert on key concepts rather than exact wording so the rule can be- # rephrased without breaking the test. The invariants are: (1) multi-row- # forms are called out somewhere in the prompt, (2) the `full_block`- # episode case is covered, (3) the prescribed action is removal, not- # accumulation.text = _PROMPT_PATH.read_text(encoding="utf-8")assert "multi-row" in text.lower()assert "full_block" in text- # The fix's whole point: REMOVE (not just dedupe) the ambiguous label- # selector. Case-sensitive — the prompt uses upper-case REMOVE as a- # directive marker.assert "REMOVE" in textassert "label:has-text" in text@@ -33,92 +27,99 @@ class TestValidateProactiveMisuse:def setup_method(self) -> None:self.reviewer = ScriptReviewer()- def test_fallback_is_fine(self) -> None:- code = """+ @pytest.mark.parametrize(+ ("cases",),+ [+ pytest.param(+ [+ (+ "fallback_is_fine",+ """async def login(page, context):await page.fill(selector='input[name="email"]', value='test', ai='fallback', prompt='email')await page.click(selector='button', ai='fallback', prompt='submit')-"""- assert self.reviewer._validate_proactive_misuse(code) is None-- def test_proactive_on_fill_flagged(self) -> None:- code = """-async def login(page, context):- await page.fill(selector='input[placeholder="Username"]', ai='proactive', prompt='username')-"""- error = self.reviewer._validate_proactive_misuse(code)- assert error is not None- assert "page.fill()" in error- assert "ai='fallback'" in error-- def test_proactive_on_click_flagged(self) -> None:- code = """-async def block_fn(page, context):- await page.click(selector='button:has-text("Submit")', ai='proactive', prompt='submit')-"""- error = self.reviewer._validate_proactive_misuse(code)- assert error is not None- assert "page.click()" in error-- def test_proactive_on_select_option_flagged(self) -> None:- code = """-async def block_fn(page, context):- await page.select_option(selector='select[name="format"]', value='PDF', ai='proactive', prompt='format')-"""- error = self.reviewer._validate_proactive_misuse(code)- assert error is not None- assert "page.select_option()" in error-- def test_proactive_on_type_flagged(self) -> None:- code = """-async def block_fn(page, context):- await page.type(selector='input[name="search"]', ai='proactive', prompt='search')-"""- error = self.reviewer._validate_proactive_misuse(code)- assert error is not None- assert "page.type()" in error-- def test_proactive_on_extract_not_flagged(self) -> None:- """ai='proactive' on extract is legitimate — extract doesn't have selectors."""- code = """+""",+ None,+ ),+ (+ "proactive_on_extract_not_flagged",+ """async def block_fn(page, context):result = await page.extract(prompt='Get the invoice data', ai='proactive')-"""- # extract is not in _INTERACTION_METHODS, so this should pass- assert self.reviewer._validate_proactive_misuse(code) is None-- def test_proactive_without_selector_not_flagged(self) -> None:- """ai='proactive' WITHOUT selector= is the documented escape hatch (SKY-9436).-- It is intentional and runtime-safe (always invokes LLM, no- `selector: expected string` crash). Only flag when paired with selector=.- """- code = """+""",+ None,+ ),+ (+ "proactive_without_selector_not_flagged",+ """async def block_fn(page, context):await page.click(ai='proactive', prompt='Click the next-step button')await page.fill(value='hello', ai='proactive', prompt='Fill the field')-"""- assert self.reviewer._validate_proactive_misuse(code) is None-- def test_selector_inside_prompt_does_not_falsely_flag(self) -> None:- """Regression: prompt text containing 'selector=' must not make a proactive call look like proactive-with-selector (CORR-3)."""- code = """+""",+ None,+ ),+ (+ "selector_inside_prompt_does_not_falsely_flag",+ """async def block_fn(page, context):await page.click(ai='proactive', prompt='No selector= available for this widget')-"""- assert self.reviewer._validate_proactive_misuse(code) is None-- def test_comments_ignored(self) -> None:- code = """+""",+ None,+ ),+ (+ "comments_ignored",+ """async def block_fn(page, context):# await page.fill(selector='input', ai='proactive', prompt='test')await page.fill(selector='input', value='x', ai='fallback', prompt='test')-"""- assert self.reviewer._validate_proactive_misuse(code) is None-- def test_multiline_call_flagged(self) -> None:- """ai='proactive' on a continuation line should still be caught."""- code = """+""",+ None,+ ),+ ],+ id="allowed-proactive-shapes",+ ),+ pytest.param(+ [+ (+ "proactive_on_fill_flagged",+ """+async def login(page, context):+ await page.fill(selector='input[placeholder="Username"]', ai='proactive', prompt='username')+""",+ ("page.fill()", "ai='fallback'"),+ ),+ (+ "proactive_on_click_flagged",+ """+async def block_fn(page, context):+ await page.click(selector='button:has-text("Submit")', ai='proactive', prompt='submit')+""",+ ("page.click()",),+ ),+ (+ "proactive_on_select_option_flagged",+ """+async def block_fn(page, context):+ await page.select_option(selector='select[name="format"]', value='PDF', ai='proactive', prompt='format')+""",+ ("page.select_option()",),+ ),+ (+ "proactive_on_type_flagged",+ """+async def block_fn(page, context):+ await page.type(selector='input[name="search"]', ai='proactive', prompt='search')+""",+ ("page.type()",),+ ),+ ],+ id="selector-plus-proactive-is-flagged",+ ),+ pytest.param(+ [+ (+ "multiline_call_flagged",+ """async def login(page, context):await page.fill(selector='input[name="email"]',@@ -126,23 +127,38 @@ async def login(page, context):ai='proactive',prompt='email field',)-"""- error = self.reviewer._validate_proactive_misuse(code)- assert error is not None- assert "page.fill()" in error-- def test_multiple_issues_reported(self) -> None:- code = """+""",+ ("page.fill()",),+ )+ ],+ id="multiline-call",+ ),+ pytest.param(+ [+ (+ "multiple_issues_reported",+ """async def login(page, context):await page.fill(selector='#user', ai='proactive', prompt='user')await page.fill(selector='#pass', ai='proactive', prompt='pass')await page.click(selector='#submit', ai='proactive', prompt='submit')-"""- error = self.reviewer._validate_proactive_misuse(code)- assert error is not None- # Should mention multiple occurrences- assert "page.fill()" in error- assert "page.click()" in error+""",+ ("page.fill()", "page.click()"),+ )+ ],+ id="multiple-issues",+ ),+ ],+ )+ def test_proactive_misuse_cases(self, cases: list[tuple[str, str, tuple[str, ...] | None]]) -> None:+ for case_id, code, expected_substrings in cases:+ error = self.reviewer._validate_proactive_misuse(code)+ if expected_substrings is None:+ assert error is None, case_id+ continue+ assert error is not None, case_id+ for expected in expected_substrings:+ assert expected in error, case_idclass TestValidateFragileSelectors:@@ -151,98 +167,122 @@ class TestValidateFragileSelectors:def setup_method(self) -> None:self.reviewer = ScriptReviewer()- def test_stable_selectors_pass(self) -> None:- code = """+ @pytest.mark.parametrize(+ ("cases",),+ [+ pytest.param(+ [+ (+ "stable_selectors_pass",+ """async def login(page, context):await page.fill(selector='input[name="email"]', value='test', ai='fallback', prompt='email')await page.click(selector='button:has-text("Sign In")', ai='fallback', prompt='sign in')await page.fill(selector='input[placeholder="Password"]', value='pass', ai='fallback', prompt='pass')-"""- assert self.reviewer._validate_fragile_selectors(code) is None-- def test_dotnetnuke_id_flagged(self) -> None:- code = """+""",+ None,+ ),+ (+ "comments_ignored",+ """+async def block_fn(page, context):+ # selector='#dnn_ctl00_aMyAccount' is fragile but this is a comment+ await page.click(selector='button:has-text("Submit")', ai='fallback', prompt='submit')+""",+ None,+ ),+ (+ "no_selector_passes",+ """+async def block_fn(page, context):+ await page.click(ai='fallback', prompt='click the submit button')+""",+ None,+ ),+ ],+ id="stable-and-ignored",+ ),+ pytest.param(+ [+ (+ "dotnetnuke_id_flagged",+ """async def login(page, context):await page.click(selector='#dnn_ctl00_aMyAccount', ai='fallback', prompt='account')-"""- error = self.reviewer._validate_fragile_selectors(code)- assert error is not None- assert "dnn_" in error- assert "DotNetNuke" in error-- def test_ember_id_flagged(self) -> None:- code = """+""",+ ("dnn_", "DotNetNuke"),+ ),+ (+ "ember_id_flagged",+ """async def block_fn(page, context):await page.click(selector='#ember-1234', ai='fallback', prompt='click')-"""- error = self.reviewer._validate_fragile_selectors(code)- assert error is not None- assert "ember" in error.lower()-- def test_react_select_id_flagged(self) -> None:- code = """+""",+ ("ember",),+ ),+ (+ "react_select_id_flagged",+ """async def block_fn(page, context):await page.click(selector='#react-select-5-option-2', ai='fallback', prompt='select option')-"""- error = self.reviewer._validate_fragile_selectors(code)- assert error is not None- assert "react-select" in error.lower()-- def test_css_in_js_class_flagged(self) -> None:- code = """+""",+ ("react-select",),+ ),+ (+ "css_in_js_class_flagged",+ """async def block_fn(page, context):await page.click(selector='.css-1a2b3c', ai='fallback', prompt='click button')-"""- error = self.reviewer._validate_fragile_selectors(code)- assert error is not None- assert "css-" in error.lower()-- def test_mui_class_flagged(self) -> None:- code = """+""",+ ("css-",),+ ),+ (+ "mui_class_flagged",+ """async def block_fn(page, context):await page.click(selector='.MuiButton-root', ai='fallback', prompt='click button')-"""- error = self.reviewer._validate_fragile_selectors(code)- assert error is not None-- def test_extjs_id_flagged(self) -> None:- code = """+""",+ (),+ ),+ (+ "extjs_id_flagged",+ """async def block_fn(page, context):await page.click(selector='#ext-gen-456', ai='fallback', prompt='click')-"""- error = self.reviewer._validate_fragile_selectors(code)- assert error is not None- assert "ext-gen" in error.lower()-- def test_comments_ignored(self) -> None:- code = """-async def block_fn(page, context):- # selector='#dnn_ctl00_aMyAccount' is fragile but this is a comment- await page.click(selector='button:has-text("Submit")', ai='fallback', prompt='submit')-"""- assert self.reviewer._validate_fragile_selectors(code) is None-- def test_multiline_call_flagged(self) -> None:- """Fragile selector on a continuation line should still be caught."""- code = """+""",+ ("ext-gen",),+ ),+ ],+ id="fragile-framework-patterns",+ ),+ pytest.param(+ [+ (+ "multiline_call_flagged",+ """async def block_fn(page, context):await page.click(selector='#dnn_ctl00_aMyAccount',ai='fallback',prompt='account link',)-"""- error = self.reviewer._validate_fragile_selectors(code)- assert error is not None- assert "dnn_" in error-- def test_no_selector_passes(self) -> None:- """Code without selector= kwargs should not trigger."""- code = """-async def block_fn(page, context):- await page.click(ai='fallback', prompt='click the submit button')-"""- assert self.reviewer._validate_fragile_selectors(code) is None+""",+ ("dnn_",),+ )+ ],+ id="multiline-selector",+ ),+ ],+ )+ def test_fragile_selector_cases(self, cases: list[tuple[str, str, tuple[str, ...] | None]]) -> None:+ for case_id, code, expected_substrings in cases:+ error = self.reviewer._validate_fragile_selectors(code)+ if expected_substrings is None:+ assert error is None, case_id+ continue+ assert error is not None, case_id+ for expected in expected_substrings:+ assert expected.lower() in error.lower(), case_idclass TestValidateHardcodedRunData:@@ -251,101 +291,111 @@ class TestValidateHardcodedRunData:def setup_method(self) -> None:self.reviewer = ScriptReviewer()- def test_clean_code_passes(self) -> None:- code = """+ @pytest.mark.parametrize(+ ("cases",),+ [+ pytest.param(+ [+ (+ "clean_code_passes",+ """async def block_fn(page, context):await page.click(selector='button:has-text("Download")', ai='fallback', prompt='download invoice')-"""- assert self.reviewer._validate_hardcoded_run_data(code) is None-- def test_date_in_selector_flagged(self) -> None:- code = """+""",+ None,+ ),+ (+ "stable_short_text_ok",+ """async def block_fn(page, context):- await page.click(selector='a:has-text("03/17/2026")', ai='fallback', prompt='click invoice')-"""- error = self.reviewer._validate_hardcoded_run_data(code)- assert error is not None- assert "03/17/2026" in error- assert "date" in error.lower()-- def test_iso_date_in_selector_flagged(self) -> None:- code = """+ await page.click(selector='button:has-text("OK")', ai='fallback', prompt='confirm')+""",+ None,+ ),+ (+ "long_has_text_ok",+ """async def block_fn(page, context):- await page.click(selector='td:has-text("2026-03-17")', ai='fallback', prompt='click invoice')-"""- error = self.reviewer._validate_hardcoded_run_data(code)- assert error is not None- assert "2026-03-17" in error-- def test_date_in_prompt_flagged(self) -> None:- code = """+ await page.click(selector='a:has-text("Download")', ai='fallback', prompt='download')+""",+ None,+ ),+ (+ "comments_ignored",+ """async def block_fn(page, context):- await page.select_option(selector='select', value='PDF', ai='proactive', prompt='Select format for invoice dated 3/17/2026')-"""- error = self.reviewer._validate_hardcoded_run_data(code)- assert error is not None- assert "3/17/2026" in error- assert "prompt" in error.lower()-- def test_short_has_text_flagged(self) -> None:- """a:has-text("6") is almost certainly hardcoded run data."""- code = """+ # Don't use selector='a:has-text("6")' — it's hardcoded+ await page.click(selector='a:has-text("Download")', ai='fallback', prompt='download')+""",+ None,+ ),+ (+ "parameterized_date_ok",+ """async def block_fn(page, context):- await page.click(selector='a:has-text("6")', ai='fallback', prompt='click invoice')-"""- error = self.reviewer._validate_hardcoded_run_data(code)- assert error is not None- assert ':has-text("6")' in error-- def test_short_has_text_number_flagged(self) -> None:- code = """+ start_date = context.parameters['download_start_date']+ await page.click(selector=f'a:has-text("{start_date}")', ai='fallback', prompt='click date')+""",+ None,+ ),+ (+ "text_patterns_without_email_ok",+ """async def block_fn(page, context):- await page.click(selector='a:has-text("12")', ai='fallback', prompt='click row')-"""- error = self.reviewer._validate_hardcoded_run_data(code)- assert error is not None-- def test_stable_short_text_ok(self) -> None:- """Common UI elements with short text like 'OK' or 'X' should pass."""- code = """+ state = await page.classify(+ options={"login": "login page", "dashboard": "dashboard"},+ text_patterns={+ "login": "Welcome, Username, Password, Sign in",+ "dashboard": "Logout, Billing & Payments, Service Management",+ },+ )+""",+ None,+ ),+ ],+ id="allowed-run-data-shapes",+ ),+ pytest.param(+ [+ (+ "date_in_selector_flagged",+ """async def block_fn(page, context):- await page.click(selector='button:has-text("OK")', ai='fallback', prompt='confirm')-"""- assert self.reviewer._validate_hardcoded_run_data(code) is None-- def test_long_has_text_ok(self) -> None:- """:has-text with meaningful text (3+ chars) should pass."""- code = """+ await page.click(selector='a:has-text("03/17/2026")', ai='fallback', prompt='click invoice')+""",+ ("03/17/2026", "date"),+ ),+ (+ "iso_date_in_selector_flagged",+ """async def block_fn(page, context):- await page.click(selector='a:has-text("Download")', ai='fallback', prompt='download')-"""- assert self.reviewer._validate_hardcoded_run_data(code) is None-- def test_comments_ignored(self) -> None:- code = """+ await page.click(selector='td:has-text("2026-03-17")', ai='fallback', prompt='click invoice')+""",+ ("2026-03-17",),+ ),+ (+ "date_in_prompt_flagged",+ """async def block_fn(page, context):- # Don't use selector='a:has-text("6")' — it's hardcoded- await page.click(selector='a:has-text("Download")', ai='fallback', prompt='download')-"""- assert self.reviewer._validate_hardcoded_run_data(code) is None-- def test_multiline_date_in_selector_flagged(self) -> None:- """Hardcoded date on a continuation line should still be caught."""- code = """+ await page.select_option(selector='select', value='PDF', ai='proactive', prompt='Select format for invoice dated 3/17/2026')+""",+ ("3/17/2026", "prompt"),+ ),+ (+ "multiline_date_in_selector_flagged",+ """async def block_fn(page, context):await page.click(selector='a:has-text("03/17/2026")',ai='fallback',prompt='click invoice',)-"""- error = self.reviewer._validate_hardcoded_run_data(code)- assert error is not None- assert "03/17/2026" in error-- def test_multiline_date_in_prompt_flagged(self) -> None:- """Hardcoded date in prompt on a continuation line should still be caught."""- code = """+""",+ ("03/17/2026",),+ ),+ (+ "multiline_date_in_prompt_flagged",+ """async def block_fn(page, context):await page.select_option(selector='select',@@ -353,24 +403,38 @@ async def block_fn(page, context):ai='proactive',prompt='Select format for invoice dated 3/17/2026',)-"""- error = self.reviewer._validate_hardcoded_run_data(code)- assert error is not None- assert "3/17/2026" in error-- def test_parameterized_date_ok(self) -> None:- """Dates in context.parameters references should not trigger."""- code = """+""",+ ("3/17/2026",),+ ),+ ],+ id="date-literals-flagged",+ ),+ pytest.param(+ [+ (+ "short_has_text_flagged",+ """async def block_fn(page, context):- start_date = context.parameters['download_start_date']- await page.click(selector=f'a:has-text("{start_date}")', ai='fallback', prompt='click date')-"""- # f-string selectors won't match the date regex in the selector value- assert self.reviewer._validate_hardcoded_run_data(code) is None-- def test_email_in_text_patterns_flagged(self) -> None:- """Email addresses in text_patterns are PII that should not be in cached scripts."""- code = """+ await page.click(selector='a:has-text("6")', ai='fallback', prompt='click invoice')+""",+ (':has-text("6")',),+ ),+ (+ "short_has_text_number_flagged",+ """+async def block_fn(page, context):+ await page.click(selector='a:has-text("12")', ai='fallback', prompt='click row')+""",+ (),+ ),+ ],+ id="short-has-text-flagged",+ ),+ pytest.param(+ [+ (+ "email_in_text_patterns_flagged",+ """async def block_fn(page, context):state = await page.classify(options={"login": "login page", "dashboard": "dashboard"},@@ -379,24 +443,23 @@ async def block_fn(page, context):"dashboard": "Logout, [email protected], John Smith, Billing",},)-"""- error = self.reviewer._validate_hardcoded_run_data(code)- assert error is not None- assert "email" in error.lower()-- def test_text_patterns_without_email_ok(self) -> None:- """Generic text_patterns without PII should pass."""- code = """-async def block_fn(page, context):- state = await page.classify(- options={"login": "login page", "dashboard": "dashboard"},- text_patterns={- "login": "Welcome, Username, Password, Sign in",- "dashboard": "Logout, Billing & Payments, Service Management",- },+""",+ ("email",),+ )+ ],+ id="text-patterns-pii",+ ),+ ],)-"""- assert self.reviewer._validate_hardcoded_run_data(code) is None+ def test_hardcoded_run_data_cases(self, cases: list[tuple[str, str, tuple[str, ...] | None]]) -> None:+ for case_id, code, expected_substrings in cases:+ error = self.reviewer._validate_hardcoded_run_data(code)+ if expected_substrings is None:+ assert error is None, case_id+ continue+ assert error is not None, case_id+ for expected in expected_substrings:+ assert expected.lower() in error.lower(), case_idclass TestValidateMissingSelectors:@@ -421,10 +484,13 @@ async def block_fn(page, context):class TestExtractCachedBlocks:"""Tests for the shared block extraction utilities."""- def test_extract_all_blocks(self):- from skyvern.services.workflow_script_service import extract_cached_blocks_from_source+ def test_extract_cached_block_cases(self) -> None:+ from skyvern.services.workflow_script_service import (+ extract_cached_blocks_from_source,+ extract_single_cached_block,+ )- source = """import skyvern+ multi_block_source = """import skyvern@skyvern.cached(cache_key = 'login')async def login(page, context):@@ -437,118 +503,74 @@ async def block_1(page, context):await page.click(selector='button')await page.complete()"""- result = extract_cached_blocks_from_source(source)+ result = extract_cached_blocks_from_source(multi_block_source)assert set(result.keys()) == {"login", "block_1"}assert "page.goto" in result["login"] # nosemgrep: incomplete-url-substring-sanitizationassert "page.click" in result["block_1"] # nosemgrep: incomplete-url-substring-sanitization- # login block should NOT contain block_1 codeassert "page.click" not in result["login"]- def test_extract_single_block(self):- from skyvern.services.workflow_script_service import extract_single_cached_block-- source = """import skyvern-[email protected](cache_key = 'login')-async def login(page, context):- await page.goto('https://example.com')--[email protected](cache_key = 'block_1')-async def block_1(page, context):- await page.click(selector='button')-"""- result = extract_single_cached_block(source, "block_1")- assert result is not None- assert "page.click" in result # nosemgrep: incomplete-url-substring-sanitization- assert "page.goto" not in result-- def test_extract_first_block(self):- from skyvern.services.workflow_script_service import extract_single_cached_block-- source = """import skyvern-[email protected](cache_key = 'login')-async def login(page, context):- await page.goto('https://example.com')--[email protected](cache_key = 'block_1')-async def block_1(page, context):- await page.click(selector='button')-"""- result = extract_single_cached_block(source, "login")- assert result is not None- assert "page.goto" in result # nosemgrep: incomplete-url-substring-sanitization- assert "page.click" not in result-- def test_extract_single_block_not_found(self):- from skyvern.services.workflow_script_service import extract_single_cached_block-- source = "@skyvern.cached(cache_key = 'login')\nasync def login(page, context): pass\n"- result = extract_single_cached_block(source, "nonexistent")- assert result is None-- def test_extract_empty_source(self):- from skyvern.services.workflow_script_service import extract_cached_blocks_from_source-+ single_block = extract_single_cached_block(multi_block_source, "block_1")+ assert single_block is not None+ assert "page.click" in single_block # nosemgrep: incomplete-url-substring-sanitization+ assert "page.goto" not in single_block++ first_block = extract_single_cached_block(multi_block_source, "login")+ assert first_block is not None+ assert "page.goto" in first_block # nosemgrep: incomplete-url-substring-sanitization+ assert "page.click" not in first_block++ assert (+ extract_single_cached_block(+ "@skyvern.cached(cache_key = 'login')\nasync def login(page, context): pass\n",+ "nonexistent",+ )+ is None+ )assert extract_cached_blocks_from_source("") == {}assert extract_cached_blocks_from_source("import skyvern\n") == {}- def test_extract_last_block_goes_to_eof(self):- from skyvern.services.workflow_script_service import extract_cached_blocks_from_source-- source = "@skyvern.cached(cache_key = 'only_block')\nasync def only(page, ctx):\n pass\n"- result = extract_cached_blocks_from_source(source)- assert "only_block" in result- assert "pass" in result["only_block"]+ last_block_source = "@skyvern.cached(cache_key = 'only_block')\nasync def only(page, ctx):\n pass\n"+ last_block = extract_cached_blocks_from_source(last_block_source)+ assert "only_block" in last_block+ assert "pass" in last_block["only_block"]class TestClassifyBlockStrategy:"""Tests for _classify_block_strategy template selection."""- def setup_method(self):- from skyvern.services.script_reviewer import ScriptReviewer-+ def setup_method(self) -> None:self.reviewer = ScriptReviewer()- def test_extraction_block(self):- """Block with page.extract() and no page.click() → extraction."""- code = "async def block(page, ctx):\n return await page.extract(prompt='...')\n"- result = self.reviewer._classify_block_strategy(existing_code=code)- assert result == "extraction"-- def test_extraction_with_click_is_sequential(self):- """Block with both extract and click → sequential (not extraction)."""- code = (- "async def block(page, ctx):\n await page.click('#btn')\n return await page.extract(prompt='...')\n"- )- result = self.reviewer._classify_block_strategy(existing_code=code)- assert result == "sequential"-- def test_existing_fill_form_stays_form_filling(self):- """Block already using page.fill_form() stays form_filling."""- code = "async def block(page, ctx):\n await page.fill_form(ctx.parameters, prompt='...')\n"- result = self.reviewer._classify_block_strategy(existing_code=code)- assert result == "form_filling"-- def test_download_block_is_sequential(self):- """Download block should be sequential, never form_filling."""- code = "async def download(page, ctx):\n await page.download_file(prompt='...')\n await page.complete()\n"- result = self.reviewer._classify_block_strategy(existing_code=code)- assert result == "sequential"-- def test_navigation_block_with_form_fields_is_sequential(self):- """Block on a page with form fields should still be sequential (not form_filling).-- Regression guard: this data (5 form fields + form keywords in goal + form- action types) would have triggered the removed heuristic. Verifies it's gone.- """- code = "async def block(page, ctx):\n await page.click('#btn')\n await page.complete()\n"- result = self.reviewer._classify_block_strategy(existing_code=code)- assert result == "sequential"-- def test_default_is_sequential(self):- """Empty code, no special signals → sequential."""- result = self.reviewer._classify_block_strategy(existing_code="")- assert result == "sequential"+ @pytest.mark.parametrize(+ ("code", "expected"),+ [+ pytest.param(+ "async def block(page, ctx):\n return await page.extract(prompt='...')\n",+ "extraction",+ id="extract-only",+ ),+ pytest.param(+ "async def block(page, ctx):\n await page.click('#btn')\n return await page.extract(prompt='...')\n",+ "sequential",+ id="extract-with-click",+ ),+ pytest.param(+ "async def block(page, ctx):\n await page.fill_form(ctx.parameters, prompt='...')\n",+ "form_filling",+ id="existing-fill-form",+ ),+ pytest.param(+ "async def download(page, ctx):\n await page.download_file(prompt='...')\n await page.complete()\n",+ "sequential",+ id="download",+ ),+ pytest.param(+ "async def block(page, ctx):\n await page.click('#btn')\n await page.complete()\n",+ "sequential",+ id="navigation-with-form-fields",+ ),+ pytest.param("", "sequential", id="default"),+ ],+ )+ def test_classify_block_strategy_cases(self, code: str, expected: str) -> None:+ assert self.reviewer._classify_block_strategy(existing_code=code) == expecteddiff --git a/tests/unit/test_script_reviewer_validators.py b/tests/unit/test_script_reviewer_validators.pyindex 8568bfa7c..4add991cc 100644--- a/tests/unit/test_script_reviewer_validators.py+++ b/tests/unit/test_script_reviewer_validators.py@@ -1,11 +1,17 @@"""Tests for ScriptReviewer validation methods."""from datetime import datetime+from types import SimpleNamespacefrom unittest.mock import AsyncMockimport pytest+from skyvern.core.script_generations.generate_script import (+ MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB,+ MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB,+)from skyvern.forge import app+from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterTypefrom skyvern.schemas.scripts import ScriptFallbackEpisodefrom skyvern.services.script_reviewer import ScriptReviewerfrom tests.unit.force_stub_app import start_forge_stub_app@@ -17,320 +23,286 @@ class TestValidateNoHardcodedValues:def setup_method(self) -> None:self.reviewer = ScriptReviewer()- def test_no_params_returns_none(self) -> None:- code = 'await page.click(selector="button")'- assert self.reviewer._validate_no_hardcoded_values(code, None) is None- assert self.reviewer._validate_no_hardcoded_values(code, {}) is None-- def test_short_values_ignored(self) -> None:- """Values shorter than 5 chars should not trigger (too many false positives)."""- code = """+ @pytest.mark.parametrize(+ ("cases",),+ [+ pytest.param(+ [+ (+ "no_params_returns_none_none",+ 'await page.click(selector="button")',+ None,+ None,+ ),+ (+ "no_params_returns_none_empty",+ 'await page.click(selector="button")',+ {},+ None,+ ),+ (+ "short_values_ignored",+ """async def block_fn(page, context):if 'yes' == 'yes':return {"next_block_label": "send_email", "branch_index": 0}-"""- params = {"flag": "yes", "ok": "true"}- assert self.reviewer._validate_no_hardcoded_values(code, params) is None-- def test_detects_hardcoded_email(self) -> None:- """Should catch a hardcoded email that should be context.parameters['recipient']."""- code = """-async def block_fn(page, context):- recipient = '[email protected]'- if recipient != "":- return {"next_block_label": "send_email", "branch_index": 0}-"""- params = {"recipient": "[email protected]"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "hardcoded" in error.lower()- assert "recipient" in error- assert "context.parameters" in error # nosemgrep: incomplete-url-substring-sanitization-- def test_detects_hardcoded_url(self) -> None:- """Should catch a hardcoded URL that should be context.parameters['website_url']."""- code = """-async def login(page, context):- await page.goto('https://portal.vendor.com/login')- await page.fill(selector='input[name="email"]', value=context.parameters['email'])-"""- params = {"website_url": "https://portal.vendor.com/login", "email": "[email protected]"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "website_url" in error-- def test_allows_context_parameters_reference(self) -> None:- """Using context.parameters['key'] should NOT trigger the validator."""- code = """+""",+ {"flag": "yes", "ok": "true"},+ None,+ ),+ (+ "allows_context_parameters_reference",+ """async def block_fn(page, context):recipient = context.parameters.get('recipient', '')if recipient != "":return {"next_block_label": "send_email", "branch_index": 0}-"""- params = {"recipient": "[email protected]"}- assert self.reviewer._validate_no_hardcoded_values(code, params) is None-- def test_allows_non_parameter_literals(self) -> None:- """String literals that aren't parameter values should be fine."""- code = """+""",+ {"recipient": "[email protected]"},+ None,+ ),+ (+ "allows_non_parameter_literals",+ """async def block_fn(page, context):await page.click(selector='button:has-text("Sign in")', ai='fallback', prompt='Click sign in')-"""- params = {"email": "[email protected]", "password": "secret123"}- assert self.reviewer._validate_no_hardcoded_values(code, params) is None-- def test_ignores_comments(self) -> None:- """Values in comments should not trigger."""- code = """+""",+ {"email": "[email protected]", "password": "secret123"},+ None,+ ),+ (+ "ignores_comments",+ """async def block_fn(page, context):# [email protected] is the test emailrecipient = context.parameters.get('recipient', '')if recipient != "":return {"next_block_label": "send_email", "branch_index": 0}-"""- params = {"recipient": "[email protected]"}- assert self.reviewer._validate_no_hardcoded_values(code, params) is None-- def test_detects_multiple_hardcoded_values(self) -> None:- """Should report multiple hardcoded values."""- code = """+""",+ {"recipient": "[email protected]"},+ None,+ ),+ (+ "allows_block_labels_and_keywords",+ """async def block_fn(page, context):+ return {"next_block_label": "send_email", "branch_index": 0}+""",+ {"account_number": "12345678"},+ None,+ ),+ ],+ id="allowed-and-early-exit",+ ),+ pytest.param(+ [+ (+ "detects_hardcoded_email",+ """+async def block_fn(page, context):+ recipient = '[email protected]'+ if recipient != "":+ return {"next_block_label": "send_email", "branch_index": 0}+""",+ {"recipient": "[email protected]"},+ ("hardcoded", "recipient", "context.parameters"),+ ),+ (+ "detects_hardcoded_url",+ """+async def login(page, context):await page.goto('https://portal.vendor.com/login')- await page.fill(selector='input', value='[email protected]')-"""- params = {- "website_url": "https://portal.vendor.com/login",- "recipient": "[email protected]",- }- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "2 hardcoded" in error-- def test_allows_block_labels_and_keywords(self) -> None:- """Common structural strings like block labels should not trigger."""- code = """+ await page.fill(selector='input[name="email"]', value=context.parameters['email'])+""",+ {"website_url": "https://portal.vendor.com/login", "email": "[email protected]"},+ ("website_url",),+ ),+ (+ "detects_multiple_hardcoded_values",+ """async def block_fn(page, context):- return {"next_block_label": "send_email", "branch_index": 0}-"""- # Even if "send_email" is technically a value somewhere, it's a block label- params = {"account_number": "12345678"}- assert self.reviewer._validate_no_hardcoded_values(code, params) is None-- # ------------------------------------------------------------------- # SKY-9295: substring matching — param values often appear embedded- # inside longer click-prompt literals like- # prompt='Should I download invoice 12/8/2025 for account 51410020?'- # rather than as standalone literals. Pre-fix the matcher used exact- # set membership (`param_value in code_literals`) and missed these.- # -------------------------------------------------------------------- def test_detects_param_embedded_in_click_prompt(self) -> None:- """Account number embedded inside a click prompt string should trigger."""- code = """+ await page.goto('https://portal.vendor.com/login')+ await page.fill(selector='input', value='[email protected]')+""",+ {+ "website_url": "https://portal.vendor.com/login",+ "recipient": "[email protected]",+ },+ ("2 hardcoded",),+ ),+ ],+ id="exact-literal-detections",+ ),+ pytest.param(+ [+ (+ "detects_param_embedded_in_click_prompt",+ """async def download_invoice(page, context):await page.click(ai='fallback',prompt='Should I download the invoice dated 12/8/2025 for account 51410020?',)-"""- params = {"account_number": "51410020", "download_start_date": "12/8/2025"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "account_number" in error- assert "download_start_date" in error-- def test_detects_single_param_embedded_in_prompt(self) -> None:- """Validator catches a substring even when only one param value is embedded."""- code = """+""",+ {"account_number": "51410020", "download_start_date": "12/8/2025"},+ ("account_number", "download_start_date"),+ ),+ (+ "detects_single_param_embedded_in_prompt",+ """async def block_fn(page, context):await page.click(prompt='Find the row for vendor CITYO49057A and click it')-"""- params = {"vendor_code": "CITYO49057A"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "vendor_code" in error-- def test_substring_match_respects_short_value_guard(self) -> None:- """Short param values still don't false-positive even with substring matching."""- # "yes" appears inside "yesterday" but is below the 5-char length guard.- code = """+""",+ {"vendor_code": "CITYO49057A"},+ ("vendor_code",),+ ),+ (+ "substring_match_respects_short_value_guard",+ """async def block_fn(page, context):await page.click(prompt='Click the yesterday button')-"""- params = {"flag": "yes"}- assert self.reviewer._validate_no_hardcoded_values(code, params) is None-- def test_substring_match_skips_comments(self) -> None:- """Embedded matches inside comments should still be ignored."""- code = """+""",+ {"flag": "yes"},+ None,+ ),+ (+ "substring_match_skips_comments",+ """async def block_fn(page, context):# Note: 51410020 is the test account number for stagingawait page.click(prompt='Click the next button')-"""- params = {"account_number": "51410020"}- assert self.reviewer._validate_no_hardcoded_values(code, params) is None-- def test_substring_match_skips_structural_literals(self) -> None:- """Substring matching only applies to prose literals (whitespace + ≥20 chars).- A short structural token like ``'next_block_label'`` should not trigger a- false positive when a param value happens to be embedded in the token."""- code = """+""",+ {"account_number": "51410020"},+ None,+ ),+ (+ "substring_match_skips_structural_literals",+ """async def block_fn(page, context):return {"next_block_label": "send_email", "branch_index": 0}-"""- # "block" (5 chars) is a substring of "next_block_label" but- # "next_block_label" is a structural keyword, not a prose literal — it- # has no whitespace and is only 16 chars. The validator must ignore it.- params = {"target": "block"}- assert self.reviewer._validate_no_hardcoded_values(code, params) is None-- def test_validator_threshold_matches_generator_threshold(self) -> None:- """Validator's minimum length must match the generator's substitution- threshold (``MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB``). A 4-char value- embedded in a prose literal is hardcoded and should be flagged."""- from skyvern.core.script_generations.generate_script import MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB-- # Sanity: keep this test honest if the generator threshold ever changes.- assert MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB == 4-- # Embed "1234" in a prose literal (whitespace + > prose-min-len).- code = """+""",+ {"target": "block"},+ None,+ ),+ ],+ id="substring-matching",+ ),+ pytest.param(+ [+ (+ "validator_threshold_matches_generator_threshold",+ """async def block_fn(page, context):await page.click(prompt='Click the box labeled 1234 to continue')-"""- params = {"order_code": "1234"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "order_code" in error-- # ------------------------------------------------------------------- # SKY-9295 v2: libcst-based literal extraction so triple-quoted f-strings- # and other multiline prompts are no longer a blind spot.- # -------------------------------------------------------------------- def test_validator_catches_hardcoded_in_triple_quoted_prompt(self) -> None:- """Generator emits triple-quoted f-strings when prompt content has- newlines or quotes (generate_script.py:547-548). The pre-libcst regex- scanner missed those. With libcst, literals from triple-quoted f-strings- are visible to substring matching."""- code = '''+""",+ {"order_code": "1234"},+ ("order_code",),+ ),+ (+ "validator_skips_oversized_param_values",+ f'''+async def block_fn(page, context):+ note = "{"x" * (MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB + 1)}"+''',+ {"huge_payload": "x" * (MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB + 1)},+ None,+ ),+ ],+ id="generator-thresholds",+ ),+ pytest.param(+ [+ (+ "validator_catches_hardcoded_in_triple_quoted_prompt",+ '''async def block_fn(page, context):await page.click(ai='fallback',prompt=f"""Find the row for account 51410020and click the download button""",)-'''- params = {"account_number": "51410020"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "account_number" in error-- def test_validator_catches_hardcoded_in_fstring_text_segment(self) -> None:- """Static text segments of an f-string (between {expr} interpolations)- are scannable; if a parameter value is baked into the text portion, the- validator must flag it."""- code = """+''',+ {"account_number": "51410020"},+ ("account_number",),+ ),+ (+ "validator_catches_hardcoded_in_fstring_text_segment",+ """async def block_fn(page, context):name = context.parameters['name']await page.click(prompt=f'Send invoice 51410020 to {name}')-"""- params = {"account_number": "51410020"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "account_number" in error-- def test_validator_skips_fstring_interpolation_expression(self) -> None:- """Interpolation expressions reference runtime values- (``context.parameters[...]``), not literals — they shouldn't be scanned- for hardcoded values. A reference to a key by string subscript inside- an interpolation must NOT be flagged."""- code = """+""",+ {"account_number": "51410020"},+ ("account_number",),+ ),+ (+ "validator_skips_fstring_interpolation_expression",+ """async def block_fn(page, context):await page.click(prompt=f"Send to {context.parameters['recipient']}")-"""- params = {"recipient": "[email protected]"}- # No literal contains "[email protected]"; only the interpolation- # references the key. Validator must not false-positive.- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is None-- # ------------------------------------------------------------------- # SKY-9295 v2: prose-literal threshold (whitespace + ≥ PROSE_LITERAL_MIN_LEN- # chars). The floor is 8 (down from a stricter 20) so short embedded- # prompts like ``'Click 51410020 now'`` are still caught.- # -------------------------------------------------------------------- def test_prose_literal_band_catches_embedded_value(self) -> None:- """A short prose literal with whitespace catches an embedded param- value via substring matching."""- code = """+""",+ {"recipient": "[email protected]"},+ None,+ ),+ ],+ id="fstring-literals",+ ),+ pytest.param(+ [+ (+ "prose_literal_band_catches_embedded_value",+ """async def block_fn(page, context):await page.click(prompt='Click 51410020 now')-"""- # "Click 51410020 now" = 18 chars, has whitespace, ≥ 8. Prose-eligible.- params = {"account_number": "51410020"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "account_number" in error-- def test_prose_literal_band_below_floor_skips(self) -> None:- """Whitespace-bearing literals shorter than the prose floor are not- eligible for substring matching (and exact-match still fails because- the literal isn't equal to the param value)."""- # "ok 1234" = 7 chars, < 8 → not prose; not equal to "1234" → not exact-match.- code = """+""",+ {"account_number": "51410020"},+ ("account_number",),+ ),+ (+ "prose_literal_band_below_floor_skips",+ """async def block_fn(page, context):await page.click(prompt='ok 1234')-"""- params = {"order_code": "1234"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is None-- def test_prose_literal_no_whitespace_skips(self) -> None:- """Mid-length literals without whitespace are not prose — they're- structural identifiers (``order_12345_id``, dict keys, etc.)."""- code = """+""",+ {"order_code": "1234"},+ None,+ ),+ (+ "prose_literal_no_whitespace_skips",+ """async def block_fn(page, context):return {"order_12345_id": "value", "branch_index": 0}-"""- params = {"order_id": "12345"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is None-- def test_prose_literal_band_exact_match_still_fires(self) -> None:- """A param value that equals an entire short literal is still flagged- via the exact-match path (which runs against every literal regardless- of prose status)."""- code = """+""",+ {"order_id": "12345"},+ None,+ ),+ (+ "prose_literal_band_exact_match_still_fires",+ """async def block_fn(page, context):await page.fill(selector="#code", value="12345")-"""- params = {"order_code": "12345"}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is not None- assert "order_code" in error-- # ------------------------------------------------------------------- # SKY-9295 v2: MAX cap aligned with generator.- # -------------------------------------------------------------------- def test_validator_skips_oversized_param_values(self) -> None:- """Param values longer than ``MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB``- are skipped — the generator wouldn't have substituted them, so the- validator must not flag them either (else generator/validator drift- creates an unfixable diagnostic loop)."""- from skyvern.core.script_generations.generate_script import MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB-- oversized = "x" * (MAX_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB + 1)- # Even though the literal contains the oversized value, it must be skipped.- code = f'''-async def block_fn(page, context):- note = "{oversized}"-'''- params = {"huge_payload": oversized}- error = self.reviewer._validate_no_hardcoded_values(code, params)- assert error is None+""",+ {"order_code": "12345"},+ ("order_code",),+ ),+ ],+ id="prose-literal-band",+ ),+ ],+ )+ def test_no_hardcoded_value_cases(+ self,+ cases: list[tuple[str, str, dict[str, str] | None, tuple[str, ...] | None]],+ ) -> None:+ assert MIN_PARAM_VALUE_LENGTH_FOR_PROMPT_SUB == 4+ for case_id, code, params, expected_substrings in cases:+ error = self.reviewer._validate_no_hardcoded_values(code, params)+ if expected_substrings is None:+ assert error is None, case_id+ continue+ assert error is not None, case_id+ for expected in expected_substrings:+ assert expected in error, case_idclass TestValidateParameterPreservation:@@ -339,78 +311,115 @@ class TestValidateParameterPreservation:def setup_method(self) -> None:self.reviewer = ScriptReviewer()- def test_no_existing_code_returns_none(self) -> None:- new_code = "await page.fill(selector='#email', ai='proactive', prompt='email')"- assert self.reviewer._validate_parameter_preservation(new_code, None, ["email"]) is None-- def test_no_parameter_keys_returns_none(self) -> None:- old_code = "await page.fill(selector='#email', value=context.parameters['email'])"- new_code = "await page.fill(selector='#email', ai='proactive', prompt='email')"- assert self.reviewer._validate_parameter_preservation(new_code, old_code, []) is None-- def test_preserved_refs_returns_none(self) -> None:- """When all parameter refs are preserved, validation passes."""- old_code = """-await page.fill(selector='#email', value=context.parameters['email'])-await page.fill(selector='#pass', value=context.parameters['password'])-"""- new_code = """+ @pytest.mark.parametrize(+ ("cases",),+ [+ pytest.param(+ [+ (+ "no_existing_code_returns_none",+ "await page.fill(selector='#email', ai='proactive', prompt='email')",+ None,+ ["email"],+ None,+ ),+ (+ "no_parameter_keys_returns_none",+ "await page.fill(selector='#email', ai='proactive', prompt='email')",+ "await page.fill(selector='#email', value=context.parameters['email'])",+ [],+ None,+ ),+ (+ "preserved_refs_returns_none",+ """choice = await page.classify(...)if choice == 0:await page.fill(selector='#email', value=context.parameters['email'])await page.fill(selector='#pass', value=context.parameters['password'])-"""- assert self.reviewer._validate_parameter_preservation(new_code, old_code, ["email", "password"]) is None-- def test_detects_dropped_refs(self) -> None:- """When the LLM drops value= refs, validation catches it."""- old_code = """+""",+ """await page.fill(selector='#email', value=context.parameters['email'])await page.fill(selector='#pass', value=context.parameters['password'])-"""- new_code = """+""",+ ["email", "password"],+ None,+ ),+ (+ "ignores_refs_not_in_parameter_keys",+ "await page.fill(selector='#x', ai='proactive', prompt='fill x')",+ "await page.fill(selector='#x', value=context.parameters['invented_key'])",+ ["email", "password"],+ None,+ ),+ ],+ id="preserved-or-unconstrained",+ ),+ pytest.param(+ [+ (+ "detects_dropped_refs",+ """choice = await page.classify(...)if choice == 0:await page.fill(selector='#email', ai='proactive', prompt='fill email')await page.fill(selector='#pass', ai='proactive', prompt='fill password')-"""- error = self.reviewer._validate_parameter_preservation(new_code, old_code, ["email", "password"])- assert error is not None- assert "email" in error- assert "password" in error- assert "dropped" in error.lower()-- def test_ignores_refs_not_in_parameter_keys(self) -> None:- """Spurious refs in old code that aren't valid keys should be ignored."""- old_code = "await page.fill(selector='#x', value=context.parameters['invented_key'])"- new_code = "await page.fill(selector='#x', ai='proactive', prompt='fill x')"- assert self.reviewer._validate_parameter_preservation(new_code, old_code, ["email", "password"]) is None-- def test_partial_drop_detected(self) -> None:- """Dropping one ref while keeping another should flag only the dropped one."""- old_code = """+""",+ """await page.fill(selector='#email', value=context.parameters['email'])await page.fill(selector='#pass', value=context.parameters['password'])-"""- new_code = """-await page.fill(selector='#email', value=context.parameters['email'])-await page.fill(selector='#pass', ai='proactive', prompt='fill password')-"""- error = self.reviewer._validate_parameter_preservation(new_code, old_code, ["email", "password"])- assert error is not None- assert "context.parameters['password']" in error- assert "context.parameters['email']" not in error-- def test_commented_ref_in_new_code_counts_as_dropped(self) -> None:- """A parameter ref only in a comment in new code should be flagged as dropped."""- old_code = "await page.fill(selector='#email', value=context.parameters['email'])"- new_code = """+""",+ ["email", "password"],+ ("email", "password", "dropped"),+ ),+ (+ "commented_ref_in_new_code_counts_as_dropped",+ """# was: context.parameters['email']await page.fill(selector='#email', ai='proactive', prompt='fill email')-"""- error = self.reviewer._validate_parameter_preservation(new_code, old_code, ["email"])- assert error is not None- assert "email" in error+""",+ "await page.fill(selector='#email', value=context.parameters['email'])",+ ["email"],+ ("email",),+ ),+ ],+ id="dropped-references",+ ),+ pytest.param(+ [+ (+ "partial_drop_detected",+ """+await page.fill(selector='#email', value=context.parameters['email'])+await page.fill(selector='#pass', ai='proactive', prompt='fill password')+""",+ """+await page.fill(selector='#email', value=context.parameters['email'])+await page.fill(selector='#pass', value=context.parameters['password'])+""",+ ["email", "password"],+ ("context.parameters['password']", "!context.parameters['email']"),+ )+ ],+ id="partial-drop",+ ),+ ],+ )+ def test_parameter_preservation_cases(+ self,+ cases: list[tuple[str, str, str | None, list[str], tuple[str, ...] | None]],+ ) -> None:+ for case_id, new_code, old_code, parameter_keys, expected_substrings in cases:+ error = self.reviewer._validate_parameter_preservation(new_code, old_code, parameter_keys)+ if expected_substrings is None:+ assert error is None, case_id+ continue+ assert error is not None, case_id+ for expected in expected_substrings:+ if expected.startswith("!"):+ assert expected[1:] not in error, case_id+ else:+ assert expected.lower() in error.lower(), case_idclass TestValidateBranchReturns:@@ -421,126 +430,167 @@ class TestValidateBranchReturns:{"original_expression": None, "next_block_label": "block_4", "is_default": True},]- def test_valid_returns_passes(self) -> None:- code = """+ @pytest.mark.parametrize(+ ("cases",),+ [+ pytest.param(+ [+ (+ "valid_returns_passes",+ """async def block_fn(page, context):if context.parameters.get('x', 0) > 0:return {"next_block_label": "block_3", "branch_index": 0}else:return {"next_block_label": "block_4", "branch_index": 1}-"""- assert ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO) is None-- def test_invalid_label_detected(self) -> None:- """Labels not in the branch definitions should be flagged."""- code = """+""",+ BRANCHES_TWO,+ None,+ ),+ (+ "none_label_valid_when_in_branches",+ """+async def block_fn(page, context):+ if context.parameters.get('x', 0) > 0:+ return {"next_block_label": "block_3", "branch_index": 0}+ else:+ return {"next_block_label": None, "branch_index": 1}+""",+ [+ {"original_expression": "x > 0", "next_block_label": "block_3", "is_default": False},+ {"original_expression": None, "next_block_label": None, "is_default": True},+ ],+ None,+ ),+ (+ "single_quotes_handled",+ """+async def block_fn(page, context):+ return {'next_block_label': 'block_3', 'branch_index': 0}+""",+ BRANCHES_TWO,+ None,+ ),+ ],+ id="valid-literals",+ ),+ pytest.param(+ [+ (+ "invalid_label_detected",+ """async def block_fn(page, context):if context.parameters.get('x', 0) > 0:return {"next_block_label": "block_99", "branch_index": 0}else:return {"next_block_label": "block_4", "branch_index": 1}-"""- error = ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO)- assert error is not None- assert "block_99" in error-- def test_invalid_index_detected(self) -> None:- """branch_index values outside 0..N-1 should be flagged."""- code = """+""",+ BRANCHES_TWO,+ ("block_99",),+ ),+ (+ "invalid_index_detected",+ """async def block_fn(page, context):if context.parameters.get('x', 0) > 0:return {"next_block_label": "block_3", "branch_index": 0}else:return {"next_block_label": "block_4", "branch_index": -1}-"""- error = ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO)- assert error is not None- assert "-1" in error-- def test_none_label_invalid_when_not_in_branches(self) -> None:- """None next_block_label should be flagged when no branch has a null target."""- code = """+""",+ BRANCHES_TWO,+ ("-1",),+ ),+ (+ "none_label_invalid_when_not_in_branches",+ """async def block_fn(page, context):if context.parameters.get('x', 0) > 0:return {"next_block_label": "block_3", "branch_index": 0}else:return {"next_block_label": None, "branch_index": 1}-"""- error = ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO)- assert error is not None- assert "next_block_label" in error-- def test_none_label_valid_when_in_branches(self) -> None:- """None next_block_label should pass when a branch has a null target."""- branches = [- {"original_expression": "x > 0", "next_block_label": "block_3", "is_default": False},- {"original_expression": None, "next_block_label": None, "is_default": True},- ]- code = """+""",+ BRANCHES_TWO,+ ("next_block_label",),+ ),+ (+ "both_label_and_index_invalid",+ """async def block_fn(page, context):- if context.parameters.get('x', 0) > 0:- return {"next_block_label": "block_3", "branch_index": 0}- else:- return {"next_block_label": None, "branch_index": 1}-"""- assert ScriptReviewer._validate_branch_returns(code, branches) is None-- def test_no_literals_passes(self) -> None:- """Code using variables (not literals) should pass — can't validate statically."""- code = """+ return {"next_block_label": "wrong_label", "branch_index": -1}+""",+ BRANCHES_TWO,+ ("wrong_label", "-1"),+ ),+ ],+ id="invalid-literals",+ ),+ pytest.param(+ [+ (+ "no_literals_passes",+ """async def block_fn(page, context):label = compute_label()idx = compute_index()return {"next_block_label": label, "branch_index": idx}-"""- assert ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO) is None-- def test_comments_ignored(self) -> None:- """Values in comments should not trigger validation."""- code = """+""",+ BRANCHES_TWO,+ None,+ ),+ (+ "comments_ignored",+ """async def block_fn(page, context):# return {"next_block_label": "block_99", "branch_index": -1}if context.parameters.get('x', 0) > 0:return {"next_block_label": "block_3", "branch_index": 0}else:return {"next_block_label": "block_4", "branch_index": 1}-"""- assert ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO) is None-- def test_empty_branches_passes(self) -> None:- assert ScriptReviewer._validate_branch_returns("return {}", []) is None-- def test_single_quotes_handled(self) -> None:- """Single-quoted keys should be parsed correctly."""- code = """-async def block_fn(page, context):- return {'next_block_label': 'block_3', 'branch_index': 0}-"""- assert ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO) is None-- def test_both_label_and_index_invalid(self) -> None:- """Both invalid label and index should be reported."""- code = """-async def block_fn(page, context):- return {"next_block_label": "wrong_label", "branch_index": -1}-"""- error = ScriptReviewer._validate_branch_returns(code, self.BRANCHES_TWO)- assert error is not None- assert "wrong_label" in error- assert "-1" in error+""",+ BRANCHES_TWO,+ None,+ ),+ ],+ id="ignored-static-unvalidated",+ ),+ pytest.param(+ [+ (+ "empty_branches_passes",+ "return {}",+ [],+ None,+ )+ ],+ id="empty-branches",+ ),+ ],+ )+ def test_branch_return_cases(+ self,+ cases: list[tuple[str, str, list[dict], tuple[str, ...] | None]],+ ) -> None:+ for case_id, code, branches, expected_substrings in cases:+ error = ScriptReviewer._validate_branch_returns(code, branches)+ if expected_substrings is None:+ assert error is None, case_id+ continue+ assert error is not None, case_id+ for expected in expected_substrings:+ assert expected in error, case_idclass TestLoadFilteredRunParamValues:- """Tests for load_filtered_run_param_values (shared loader for 3 reviewer ingestion sites)."""+ """Tests for load_filtered_run_param_values."""- def _make_workflow_param(self, key: str) -> object:- from datetime import datetime-- from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType+ def _make_workflow_param(+ self, key: str, parameter_type: WorkflowParameterType = WorkflowParameterType.STRING+ ) -> object:+ from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterreturn WorkflowParameter(workflow_parameter_id=f"wp_{key}",- workflow_parameter_type=WorkflowParameterType.STRING,+ workflow_parameter_type=parameter_type,workflow_id="w_1",key=key,created_at=datetime(2026, 1, 1),@@ -548,8 +598,6 @@ class TestLoadFilteredRunParamValues:)def _make_credential_param(self, key: str) -> object:- from datetime import datetime-from skyvern.forge.sdk.workflow.models.parameter import CredentialParameterreturn CredentialParameter(@@ -561,129 +609,127 @@ class TestLoadFilteredRunParamValues:modified_at=datetime(2026, 1, 1),)- async def _run_with_params(self, monkeypatch: object, param_tuples: list) -> dict[str, str]:+ async def _run_with_params(self, monkeypatch: pytest.MonkeyPatch, param_tuples: list) -> dict[str, str]:from skyvern.services.script_reviewer import load_filtered_run_param_valuesasync def fake_get(workflow_run_id: str) -> list:return param_tuples- # Patch DATABASE.workflow_runs.get_workflow_run_parameters- from skyvern.forge import app-- monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run_parameters", fake_get) # type: ignore[attr-defined]+ monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run_parameters", fake_get)return await load_filtered_run_param_values("wr_test")+ @pytest.mark.parametrize(+ ("case_id", "param_tuples", "expected"),+ [+ pytest.param(+ "filters_sensitive_credential_param",+ "credential-param",+ {"account_number": "51410020"},+ id="credential-param-filtered",+ ),+ pytest.param(+ "filters_credential_id_subtype",+ "credential-id-subtype",+ {"account_number": "51410020"},+ id="credential-id-filtered",+ ),+ pytest.param("skips_blank_values", "blank-values", {"real": "51410020"}, id="blank-values-skipped"),+ ],+ )@pytest.mark.asyncio- async def test_filters_sensitive_credential_param(self, monkeypatch: object) -> None:- from types import SimpleNamespace-- wf_param_secret = self._make_credential_param("login")- wf_param_normal = self._make_workflow_param("account_number")- param_tuples = [- (wf_param_secret, SimpleNamespace(value="cred_xyz")),- (wf_param_normal, SimpleNamespace(value="51410020")),- ]- result = await self._run_with_params(monkeypatch, param_tuples)- assert result == {"account_number": "51410020"}-- @pytest.mark.asyncio- async def test_filters_credential_id_subtype(self, monkeypatch: object) -> None:- from datetime import datetime- from types import SimpleNamespace-- from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType-- wf_param_credential_id = WorkflowParameter(- workflow_parameter_id="wp_cred",- workflow_parameter_type=WorkflowParameterType.CREDENTIAL_ID,- workflow_id="w_1",- key="selected_credential",- created_at=datetime(2026, 1, 1),- modified_at=datetime(2026, 1, 1),- )- wf_param_normal = self._make_workflow_param("account_number")- param_tuples = [- (wf_param_credential_id, SimpleNamespace(value="cred_abc")),- (wf_param_normal, SimpleNamespace(value="51410020")),- ]- result = await self._run_with_params(monkeypatch, param_tuples)- assert "selected_credential" not in result- assert result == {"account_number": "51410020"}+ async def test_load_filtered_run_param_value_cases(+ self,+ monkeypatch: pytest.MonkeyPatch,+ case_id: str,+ param_tuples: str,+ expected: dict[str, str],+ ) -> None:+ if param_tuples == "credential-param":+ tuples = [+ (self._make_credential_param("login"), SimpleNamespace(value="cred_xyz")),+ (self._make_workflow_param("account_number"), SimpleNamespace(value="51410020")),+ ]+ elif param_tuples == "credential-id-subtype":+ tuples = [+ (+ self._make_workflow_param("selected_credential", WorkflowParameterType.CREDENTIAL_ID),+ SimpleNamespace(value="cred_abc"),+ ),+ (self._make_workflow_param("account_number"), SimpleNamespace(value="51410020")),+ ]+ else:+ tuples = [+ (self._make_workflow_param("blank"), SimpleNamespace(value=" ")),+ (self._make_workflow_param("none_val"), SimpleNamespace(value=None)),+ (self._make_workflow_param("real"), SimpleNamespace(value="51410020")),+ ]+ result = await self._run_with_params(monkeypatch, tuples)+ assert result == expected, case_id@pytest.mark.asyncio- async def test_db_failure_returns_empty(self, monkeypatch: object) -> None:- from skyvern.forge import app+ async def test_db_failure_returns_empty(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.services.script_reviewer import load_filtered_run_param_valuesasync def fake_get_raises(workflow_run_id: str) -> list:raise RuntimeError("DB connection lost")- monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run_parameters", fake_get_raises) # type: ignore[attr-defined]+ monkeypatch.setattr(app.DATABASE.workflow_runs, "get_workflow_run_parameters", fake_get_raises)result = await load_filtered_run_param_values("wr_test")assert result == {}- @pytest.mark.asyncio- async def test_skips_blank_values(self, monkeypatch: object) -> None:- from types import SimpleNamespace-- wf_param_a = self._make_workflow_param("blank")- wf_param_b = self._make_workflow_param("none_val")- wf_param_c = self._make_workflow_param("real")- param_tuples = [- (wf_param_a, SimpleNamespace(value=" ")), # whitespace-only- (wf_param_b, SimpleNamespace(value=None)),- (wf_param_c, SimpleNamespace(value="51410020")),- ]- result = await self._run_with_params(monkeypatch, param_tuples)- assert result == {"real": "51410020"}-class TestFilterSelfOutputParamKeys:- """Tests for _filter_self_output_param_keys.-- Each block carries an auto-generated ``<block_label>_output`` parameter.- Inside the block's own cached function that slot is ``None`` until the- block has finished running, so any ``context.parameters['<self>_output']``- reference crashes with ``'NoneType' object is not subscriptable``.- Upstream and downstream block outputs are valid references — only the- *current* block's output must be filtered.- """-- def test_drops_self_output_only(self) -> None:- keys = ["account_number", "Login_block_output", "Search_block_output"]- filtered = ScriptReviewer._filter_self_output_param_keys(keys, "Login_block")- assert filtered == ["account_number", "Search_block_output"]-- def test_keeps_upstream_outputs(self) -> None:- """Upstream `_output` keys are legitimate cross-block references and must survive."""- keys = ["Search_results_output", "Login_block_output"]- filtered = ScriptReviewer._filter_self_output_param_keys(keys, "Login_block")- assert "Search_results_output" in filtered- assert "Login_block_output" not in filtered-- def test_keeps_downstream_outputs(self) -> None:- """Downstream `_output` keys aren't read by the current block in practice,- but filtering only the self-output keeps the rule narrow and predictable."""- keys = ["Login_block_output", "Final_extract_output"]- filtered = ScriptReviewer._filter_self_output_param_keys(keys, "Login_block")- assert "Final_extract_output" in filtered- assert "Login_block_output" not in filtered-- def test_empty_block_label_is_noop(self) -> None:- """An empty block_label means we don't know what to filter — pass through."""- keys = ["a", "b_output"]- assert ScriptReviewer._filter_self_output_param_keys(keys, "") == keys-- def test_no_self_output_present(self) -> None:- keys = ["account_number", "username"]- assert ScriptReviewer._filter_self_output_param_keys(keys, "Login_block") == keys-- def test_partial_match_not_filtered(self) -> None:- """`block_output_other` matches `<block>_output` as a prefix but isn't the- self-output key. The filter must be exact-match, not prefix-match."""- keys = ["Login_block_output_other", "Login_block_output"]- filtered = ScriptReviewer._filter_self_output_param_keys(keys, "Login_block")- assert filtered == ["Login_block_output_other"]+ """Tests for _filter_self_output_param_keys."""++ @pytest.mark.parametrize(+ ("cases",),+ [+ pytest.param(+ [+ (+ "drops_self_output_only",+ ["account_number", "Login_block_output", "Search_block_output"],+ "Login_block",+ ["account_number", "Search_block_output"],+ ),+ (+ "keeps_upstream_outputs",+ ["Search_results_output", "Login_block_output"],+ "Login_block",+ ["Search_results_output"],+ ),+ (+ "keeps_downstream_outputs",+ ["Login_block_output", "Final_extract_output"],+ "Login_block",+ ["Final_extract_output"],+ ),+ (+ "partial_match_not_filtered",+ ["Login_block_output_other", "Login_block_output"],+ "Login_block",+ ["Login_block_output_other"],+ ),+ ],+ id="self-output-filtered-exactly",+ ),+ pytest.param(+ [+ ("empty_block_label_is_noop", ["a", "b_output"], "", ["a", "b_output"]),+ (+ "no_self_output_present",+ ["account_number", "username"],+ "Login_block",+ ["account_number", "username"],+ ),+ ],+ id="no-filter-needed",+ ),+ ],+ )+ def test_filter_self_output_param_key_cases(self, cases: list[tuple[str, list[str], str, list[str]]]) -> None:+ for case_id, keys, block_label, expected in cases:+ assert ScriptReviewer._filter_self_output_param_keys(keys, block_label) == expected, case_idclass TestValidateParameterReferencesSelfOutput:@@ -692,246 +738,184 @@ class TestValidateParameterReferencesSelfOutput:def setup_method(self) -> None:self.reviewer = ScriptReviewer()- def test_self_output_reference_rejected(self) -> None:- """When `parameter_keys` excludes the self-output, a `context.parameters['<self>_output']`- reference in the LLM output must be rejected by the validator."""- code = """+ @pytest.mark.parametrize(+ ("cases",),+ [+ pytest.param(+ [+ (+ "self_output_reference_rejected",+ """async def block_fn(page, context):await page.fill(selector='#user', value=context.parameters['Login_block_output']['username'])-"""- # parameter_keys does NOT contain Login_block_output (already filtered)- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- def test_self_output_error_message_explains_why(self) -> None:- """The retry hint should specifically explain the self-output crash so the LLM- doesn't reintroduce the same pattern on the next attempt."""- code = "value=context.parameters['Login_block_output']"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "None" in error # message should mention the None / NoneType crash mode-- def test_validator_unchanged_without_self_output_key(self) -> None:- """Existing callers that don't pass `self_output_key` keep the prior behavior."""- code = "value=context.parameters['fake_key']"- error = self.reviewer._validate_parameter_references(code, parameter_keys=["real_key"])- assert error is not None- assert "fake_key" in error- # No self-output augmentation when caller didn't opt in.- assert "self-output" not in error.lower()-- # ------------------------------------------------------------------- # Empty-valid-set edge case: when the workflow's only declared parameter- # was the block's own output (filtered → empty parameter_keys), the- # validator must still scan to catch a self-output ref.- # -------------------------------------------------------------------- def test_self_output_rejected_with_empty_parameter_keys(self) -> None:- """When parameter_keys is empty BUT self_output_key is set, the- validator must still reject self-output references — otherwise the bug- slips through any workflow whose only parameter was the block's own- output."""- code = "value=context.parameters['Login_block_output']['username']"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=[],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- def test_no_constraints_returns_none(self) -> None:- """No parameter_keys AND no self_output_key → no validation possible."""- code = "value=context.parameters['anything']"- assert self.reviewer._validate_parameter_references(code, parameter_keys=[]) is None-- def test_empty_valid_set_allows_unknown_non_self_refs(self) -> None:- """Empty ``parameter_keys`` + ``self_output_key`` in scope: the reviewer- rejects only the self-output ref. Other unknown refs may be runtime-- valid synthesized keys (``GeneratedWorkflowParameters`` field names)- that the reviewer doesn't load into its valid set, so we let them- through rather than risking false-positive rejection of legitimate- deterministic-named parameters.- """- code = "value=context.parameters['some_other_key']"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=[],- self_output_key="Login_block_output",- )- assert error is None-- # ------------------------------------------------------------------- # `.get('key')` access: regex must catch this form too — the reviewer- # prompt's example output uses ``.get(...)``, so the LLM can emit it.- # -------------------------------------------------------------------- def test_get_access_self_output_rejected(self) -> None:- """`context.parameters.get('<self>_output')` must be caught the same as- the subscript form."""- code = "value=context.parameters.get('Login_block_output')"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- def test_get_access_with_default_self_output_rejected(self) -> None:- """The `.get(key, default)` two-argument form is also caught."""- code = "value=context.parameters.get('Login_block_output', '')"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- def test_get_access_invented_key_rejected(self) -> None:- """Generic invented-key detection also applies via the `.get()` regex."""- code = "value=context.parameters.get('made_up_field')"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- )- assert error is not None- assert "made_up_field" in error-- # ------------------------------------------------------------------- # Whitespace inside subscript brackets / `.get()` parens. Reformatted- # variants like ``[ 'key' ]`` must not bypass the validator.- # -------------------------------------------------------------------- def test_spaced_subscript_self_output_rejected(self) -> None:- """`context.parameters[ 'X_output' ]['username']` must be caught despite- whitespace inside the outer subscript brackets."""- code = "value=context.parameters[ 'Login_block_output' ]['username']"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- def test_spaced_get_access_self_output_rejected(self) -> None:- """`.get( 'X_output' )` form with whitespace must also be caught."""- code = "value=context.parameters.get( 'Login_block_output' )"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- # ------------------------------------------------------------------- # Multiline access: ``ruff format`` can wrap long subscripts across- # multiple lines. The validator must still match.- # -------------------------------------------------------------------- def test_multiline_subscript_self_output_rejected(self) -> None:- """A subscript split across multiple lines must still be caught."""- code = "value=context.parameters[\n 'Login_block_output'\n]['username']"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- def test_multiline_get_access_self_output_rejected(self) -> None:- """A `.get(...)` call split across multiple lines must still be caught."""- code = "value=context.parameters.get(\n 'Login_block_output',\n {},\n)"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- # ------------------------------------------------------------------- # Whitespace between ``parameters`` and the accessor: valid Python- # (``context.parameters [...]``, ``context.parameters .get(...)``)- # must be matched too.- # -------------------------------------------------------------------- def test_space_before_subscript_self_output_rejected(self) -> None:- """`context.parameters ['X_output']` (space between `parameters` and `[`)- must be caught."""- code = "value=context.parameters ['Login_block_output']"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- def test_space_before_get_self_output_rejected(self) -> None:- """`context.parameters .get('X_output')` (space between `parameters` and- `.`) must be caught."""- code = "value=context.parameters .get('Login_block_output')"- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- self_output_key="Login_block_output",- )- assert error is not None- assert "Login_block_output" in error-- def test_multiline_ref_inside_comment_block_not_flagged(self) -> None:- """Multiline scan must still skip refs that live inside comment lines.- Each comment line is replaced with a blank line before scanning, so a- block of comment-prefixed lines containing a fake ref must not be- flagged as a violation."""- code = (- "# context.parameters['fake_key']\n"- "# context.parameters.get('another_fake_key')\n"- "value=context.parameters['account_number']\n"- )- error = self.reviewer._validate_parameter_references(- code,- parameter_keys=["account_number"],- )- assert error is None+""",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ (+ "self_output_error_message_explains_why",+ "value=context.parameters['Login_block_output']",+ ["account_number"],+ "Login_block_output",+ ("None",),+ ),+ (+ "self_output_rejected_with_empty_parameter_keys",+ "value=context.parameters['Login_block_output']['username']",+ [],+ "Login_block_output",+ ("Login_block_output",),+ ),+ ],+ id="self-output-rejected",+ ),+ pytest.param(+ [+ (+ "validator_unchanged_without_self_output_key",+ "value=context.parameters['fake_key']",+ ["real_key"],+ None,+ ("fake_key", "!self-output"),+ ),+ (+ "get_access_invented_key_rejected",+ "value=context.parameters.get('made_up_field')",+ ["account_number"],+ None,+ ("made_up_field",),+ ),+ ],+ id="invented-keys",+ ),+ pytest.param(+ [+ (+ "no_constraints_returns_none",+ "value=context.parameters['anything']",+ [],+ None,+ None,+ ),+ (+ "empty_valid_set_allows_unknown_non_self_refs",+ "value=context.parameters['some_other_key']",+ [],+ "Login_block_output",+ None,+ ),+ (+ "multiline_ref_inside_comment_block_not_flagged",+ (+ "# context.parameters['fake_key']\n"+ "# context.parameters.get('another_fake_key')\n"+ "value=context.parameters['account_number']\n"+ ),+ ["account_number"],+ None,+ None,+ ),+ ],+ id="allowed-reference-shapes",+ ),+ pytest.param(+ [+ (+ "get_access_self_output_rejected",+ "value=context.parameters.get('Login_block_output')",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ (+ "get_access_with_default_self_output_rejected",+ "value=context.parameters.get('Login_block_output', '')",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ (+ "spaced_subscript_self_output_rejected",+ "value=context.parameters[ 'Login_block_output' ]['username']",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ (+ "spaced_get_access_self_output_rejected",+ "value=context.parameters.get( 'Login_block_output' )",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ (+ "multiline_subscript_self_output_rejected",+ "value=context.parameters[\n 'Login_block_output'\n]['username']",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ (+ "multiline_get_access_self_output_rejected",+ "value=context.parameters.get(\n 'Login_block_output',\n {},\n)",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ (+ "space_before_subscript_self_output_rejected",+ "value=context.parameters ['Login_block_output']",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ (+ "space_before_get_self_output_rejected",+ "value=context.parameters .get('Login_block_output')",+ ["account_number"],+ "Login_block_output",+ ("Login_block_output",),+ ),+ ],+ id="self-output-access-variants",+ ),+ ],+ )+ def test_parameter_reference_cases(+ self,+ cases: list[tuple[str, str, list[str], str | None, tuple[str, ...] | None]],+ ) -> None:+ for case_id, code, parameter_keys, self_output_key, expected_substrings in cases:+ error = self.reviewer._validate_parameter_references(+ code,+ parameter_keys=parameter_keys,+ self_output_key=self_output_key,+ )+ if expected_substrings is None:+ assert error is None, case_id+ continue+ assert error is not None, case_id+ for expected in expected_substrings:+ if expected.startswith("!"):+ assert expected[1:] not in error.lower(), case_id+ else:+ assert expected in error, case_idclass TestConditionalCodeSelfOutputRejection:- """The conditional-review path also persists code via- ``create_script_version_from_review``, so it must run the same self-output- parameter validation as ``_review_block_internal``."""+ """The conditional-review path must reject self-output parameter references."""def setup_method(self) -> None:self.reviewer = ScriptReviewer()@pytest.mark.asyncio- async def test_generate_conditional_code_calls_validator_with_self_output(self, monkeypatch: object) -> None:- """End-to-end wiring: drive ``_generate_conditional_code`` with a mocked- LLM that returns code containing the self-output pattern, and assert- ``_validate_parameter_references`` was invoked with the block's- ``self_output_key``. A future refactor that drops the validator call- or threads the wrong ``block_label`` will make this test fail."""+ async def test_generate_conditional_code_calls_validator_with_self_output(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:start_forge_stub_app()block_label = "Branch_block"-- # Conditional code containing the self-output crash pattern. The mocked- # LLM returns this twice (max_attempts=2) so the validator's rejection- # path runs both attempts.bad_code = ("async def block_fn(page, context):\n"" if context.parameters['Branch_block_output']['flag']:\n"@@ -945,7 +929,6 @@ class TestConditionalCodeSelfOutputRejection:app.SCRIPT_REVIEWER_LLM_API_HANDLER = AsyncMock(side_effect=fake_llm_handler)- # Spy on the validator to confirm the wiring threads `self_output_key`.validator_calls: list[dict[str, object]] = []original_validator = self.reviewer._validate_parameter_references@@ -957,8 +940,6 @@ class TestConditionalCodeSelfOutputRejection:monkeypatch.setattr(self.reviewer, "_validate_parameter_references", spy_validator)- # Build a minimal episode with a branch expression so- # `_generate_conditional_code` proceeds past its early returns.episode = ScriptFallbackEpisode(episode_id="ep_1",organization_id="o_test",@@ -996,25 +977,14 @@ class TestConditionalCodeSelfOutputRejection:all_parameter_keys=["account_number", f"{block_label}_output"],)- # Validator must have been called at least once with the correct- # self_output_key for this block. This is the wiring assertion.assert validator_calls, "validator was never invoked — wiring broken"assert all(call["self_output_key"] == f"{block_label}_output" for call in validator_calls), (f"validator received wrong self_output_key: {validator_calls}")-- # The bad_code references the block's self-output, so the validator- # rejects it on every attempt and the function returns None (no code- # accepted for persistence). This confirms the rejection chain reaches- # the persistence gate.assert result is None, "self-output ref should have been rejected before persistence"@pytest.mark.asyncio- async def test_review_conditional_blocks_threads_parameter_keys(self, monkeypatch: object) -> None:- """Outer-wiring test: ``review_conditional_blocks`` loads workflow- context internally and threads ``all_parameter_keys`` into- ``_generate_conditional_code``. A refactor that drops that load (or- threads the wrong keys) must fail this test."""+ async def test_review_conditional_blocks_threads_parameter_keys(self, monkeypatch: pytest.MonkeyPatch) -> None:start_forge_stub_app()block_label = "Branch_block"@@ -1024,22 +994,18 @@ class TestConditionalCodeSelfOutputRejection:app.SCRIPT_REVIEWER_LLM_API_HANDLER = AsyncMock(side_effect=fake_llm_handler)- # Spy on `_load_workflow_context` to confirm it's called from the- # conditional path AND to control what it returns.load_calls: list[dict[str, str]] = []async def fake_load(organization_id: str, workflow_permanent_id: str) -> tuple:load_calls.append({"organization_id": organization_id, "workflow_permanent_id": workflow_permanent_id})return (- {block_label: ""}, # goals- ["account_number", f"{block_label}_output"], # all_parameter_keys- {}, # block_criteria+ {block_label: ""},+ ["account_number", f"{block_label}_output"],+ {},)monkeypatch.setattr(self.reviewer, "_load_workflow_context", fake_load)- # Spy on the validator to confirm `_generate_conditional_code` was- # called WITH the keys threaded through `_load_workflow_context`.validator_calls: list[dict[str, object]] = []original_validator = self.reviewer._validate_parameter_references@@ -1078,11 +1044,9 @@ class TestConditionalCodeSelfOutputRejection:run_parameter_values=None,)- # Assertion 1: workflow context was loaded by review_conditional_blocks itself.assert load_calls == [{"organization_id": "o_test", "workflow_permanent_id": "wpid_test"}], ("review_conditional_blocks did not load workflow context — the wiring is broken")- # Assertion 2: the validator received the loaded keys, with the self-output filtered out.assert validator_calls, "validator was never invoked from the conditional path"for call in validator_calls:assert "account_number" in call["param_keys"], "upstream key should have been threaded through"@@ -1090,8 +1054,6 @@ class TestConditionalCodeSelfOutputRejection:assert call["self_output_key"] == f"{block_label}_output", "self_output_key not threaded correctly"def test_validator_directly_rejects_self_output(self) -> None:- """Direct unit-level assertion (kept alongside the integration test):- validator returns an error message identifying the self-output key."""block_label = "Branch_block"code_with_self_output = ("async def block_fn(page, context):\n"