Skip to content

Automate browser based workflows with AI

10.7 KB
Raw Blame
"""Tests for the context.parameters reference guard (SKY-8965 Phase 1)."""

from __future__ import annotations

import pytest

from skyvern.core.script_generations.generate_script import (
    ScriptBlockSource,
    _collect_declared_param_keys,
    _collect_upstream_schema_keys,
    generate_workflow_script_python_code,
)
from skyvern.core.script_generations.parameter_reference_guard import (
    HallucinatedParameterError,
    log_or_raise_guard_result,
    validate_context_parameter_refs,
)

SCRIPT_WITH_TWO_REFS = """
async def block_fn(page, context):
    await page.fill(value=context.parameters['search_term'])
    await page.fill(value=context.parameters['other_key'])
"""

SCRIPT_WITH_PHANTOM = """
async def block_fn(page, context):
    await page.fill(value=context.parameters['preprint_search_term'])
"""

SCRIPT_WITH_COMMENT = """
# comment reference: context.parameters['ignored']
await page.fill(value=context.parameters['real'])
"""


def test_guard_passes_when_all_refs_declared() -> None:
    result = validate_context_parameter_refs(
        code=SCRIPT_WITH_TWO_REFS,
        declared_param_keys=frozenset({"search_term", "other_key"}),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    assert result.valid
    assert result.undeclared_refs == []


def test_guard_accepts_via_synthesized_keys() -> None:
    """Phase 1: synthesized `GeneratedWorkflowParameters` fields count as valid."""
    result = validate_context_parameter_refs(
        code=SCRIPT_WITH_PHANTOM,
        declared_param_keys=frozenset(),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset({"preprint_search_term"}),
    )
    assert result.valid


def test_guard_accepts_via_upstream_schema() -> None:
    code = "value=context.parameters['invoice_date']"
    result = validate_context_parameter_refs(
        code=code,
        declared_param_keys=frozenset(),
        upstream_schema_keys=frozenset({"invoice_date"}),
        synthesized_keys=frozenset(),
    )
    assert result.valid


def test_guard_detects_phantom_param_when_nothing_covers_it() -> None:
    result = validate_context_parameter_refs(
        code=SCRIPT_WITH_PHANTOM,
        declared_param_keys=frozenset(),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    assert not result.valid
    assert len(result.undeclared_refs) == 1
    assert result.undeclared_refs[0].key == "preprint_search_term"


def test_guard_skips_references_in_comments() -> None:
    result = validate_context_parameter_refs(
        code=SCRIPT_WITH_COMMENT,
        declared_param_keys=frozenset({"real"}),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    assert result.valid, "commented-out reference must not count"


def test_guard_collects_multiple_undeclared_refs() -> None:
    code = """
    await page.fill(value=context.parameters['a'])
    await page.fill(value=context.parameters['b'])
    await page.fill(value=context.parameters['c'])
    """
    result = validate_context_parameter_refs(
        code=code,
        declared_param_keys=frozenset({"a"}),
        upstream_schema_keys=frozenset({"b"}),
        synthesized_keys=frozenset(),
    )
    assert not result.valid
    assert [r.key for r in result.undeclared_refs] == ["c"]


def test_guard_catches_get_access_form() -> None:
    """Regex matches both subscript and `.get()` access patterns."""
    code = """
    a = context.parameters.get('ok_key')
    b = context.parameters.get("with_default", "fallback")
    c = context.parameters['also_ok']
    d = context.parameters.get('phantom')
    """
    result = validate_context_parameter_refs(
        code=code,
        declared_param_keys=frozenset({"ok_key", "with_default", "also_ok"}),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    assert not result.valid
    assert [r.key for r in result.undeclared_refs] == ["phantom"]


def test_guard_handles_double_and_single_quotes() -> None:
    code = """
    x = context.parameters["double"]
    y = context.parameters['single']
    """
    result = validate_context_parameter_refs(
        code=code,
        declared_param_keys=frozenset({"double", "single"}),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    assert result.valid


def test_format_error_contains_invalid_and_valid_keys() -> None:
    result = validate_context_parameter_refs(
        code="context.parameters['phantom']",
        declared_param_keys=frozenset({"real_key"}),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    msg = result.format_error()
    assert "'phantom'" in msg
    assert "'real_key'" in msg
    assert "SKY-8965" in msg


# --- log_or_raise_guard_result --------------------------------------------


def test_log_or_raise_noop_on_valid_result() -> None:
    result = validate_context_parameter_refs(
        code="no refs here",
        declared_param_keys=frozenset(),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    # Should not raise even with raise_on_violation=True
    log_or_raise_guard_result(result, raise_on_violation=True)


def test_log_or_raise_does_not_raise_phase_1() -> None:
    """Phase 1 behaviour: raise_on_violation=False → log only.

    The log assertion is verified manually via Datadog in production — structlog
    doesn't propagate to pytest's caplog without extra fixture setup, so we
    assert the no-raise contract here and rely on integration / production
    observability for the log payload.
    """
    result = validate_context_parameter_refs(
        code="context.parameters['phantom']",
        declared_param_keys=frozenset(),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    # Does not raise
    log_or_raise_guard_result(
        result,
        raise_on_violation=False,
        workflow_permanent_id="wpid_test",
        workflow_run_id="wr_test",
    )


def test_log_or_raise_raises_phase_2() -> None:
    """Phase 2 behaviour: raise_on_violation=True → throws HallucinatedParameterError."""
    result = validate_context_parameter_refs(
        code="context.parameters['phantom']",
        declared_param_keys=frozenset(),
        upstream_schema_keys=frozenset(),
        synthesized_keys=frozenset(),
    )
    with pytest.raises(HallucinatedParameterError) as exc_info:
        log_or_raise_guard_result(result, raise_on_violation=True)
    assert "'phantom'" in str(exc_info.value)
    assert exc_info.value.result is result


def test_collect_declared_params_includes_all_parameter_types() -> None:
    workflow = {
        "workflow_definition": {
            "parameters": [
                {"parameter_type": "workflow", "key": "search_term"},
                {"parameter_type": "output", "key": "extracted_date"},
                {"parameter_type": "context", "key": "loop_var"},
                {"parameter_type": "aws_secret", "key": "api_token"},
            ]
        }
    }
    keys = _collect_declared_param_keys(workflow)
    assert keys == frozenset({"search_term", "extracted_date", "loop_var", "api_token"})


def test_collect_declared_params_returns_empty_on_non_dict_definition() -> None:
    assert _collect_declared_param_keys({"workflow_definition": "not a dict"}) == frozenset()
    assert _collect_declared_param_keys({}) == frozenset()


def test_collect_upstream_schema_keys_parses_json_string_schema() -> None:
    blocks = [{"data_schema": '{"properties": {"invoice_date": {"type": "string"}, "total": {"type": "number"}}}'}]
    keys = _collect_upstream_schema_keys(blocks)
    assert keys == frozenset({"invoice_date", "total"})


def test_collect_upstream_schema_keys_ignores_invalid_json_string_schema() -> None:
    blocks = [{"data_schema": "this is not json"}]
    assert _collect_upstream_schema_keys(blocks) == frozenset()


CARRIED_FORWARD_FIELD = "what_is_the_full_name_of_the_person_to_search_for"

CACHED_BLOCK_CODE = f'''@skyvern.cached(cache_key="search_person")
async def search_person(page, context):
    await page.fill(
        selector="xpath=//input[@id='q']",
        value=context.parameters["{CARRIED_FORWARD_FIELD}"],
        prompt="What is the full name of the person to search for?",
    )
'''


@pytest.mark.asyncio
async def test_carried_forward_cached_block_does_not_fail_codegen() -> None:
    """SKY-13946: a cached block reused verbatim must not trip the guard.

    The block's actions did not run this time, so the schema this run synthesizes
    cannot declare the field its cached code references — enforcing against it
    failed codegen on every replay and permanently blocked caching.
    """
    workflow = {
        "workflow_permanent_id": "wpid_test",
        "title": "people search",
        "workflow_definition": {
            "parameters": [
                {"key": key, "parameter_type": "workflow", "workflow_parameter_type": "string"}
                for key in ("firstName", "lastName")
            ]
        },
    }
    cached_blocks = {
        "search_person": ScriptBlockSource(
            label="search_person",
            code=CACHED_BLOCK_CODE,
            run_signature="await search_person(page, context)",
            workflow_run_id="wr_previous",
            workflow_run_block_id="wrb_1",
            input_fields=[CARRIED_FORWARD_FIELD],
        )
    }

    result = await generate_workflow_script_python_code(
        file_name="main.py",
        workflow_run_request={"parameters": {}},
        workflow=workflow,
        blocks=[
            {
                "block_type": "task",
                "label": "review_results",
                "task_id": "tsk_review",
                "workflow_run_block_id": "wrb_2",
                "navigation_goal": "Open the first result",
                "url": "https://example.com/results",
            }
        ],
        actions_by_task={
            "tsk_review": [
                {
                    "action_type": "click",
                    "action_id": "act_1",
                    "xpath": "//a[1]",
                    "intention": "Open the first result",
                }
            ]
        },
        cached_blocks=cached_blocks,
        updated_block_labels={"review_results"},
    )

    assert CARRIED_FORWARD_FIELD in result.source_code


def test_collect_upstream_schema_keys_recurses_into_loop_blocks() -> None:
    blocks = [
        {
            "block_type": "for_loop",
            "loop_blocks": [
                {
                    "data_schema": {"properties": {"nested_invoice_id": {"type": "string"}}},
                },
            ],
        },
        {"data_schema": {"properties": {"outer_key": {"type": "string"}}}},
    ]
    keys = _collect_upstream_schema_keys(blocks)
    assert keys == frozenset({"nested_invoice_id", "outer_key"})