Skip to content

Automate browser based workflows with AI

feat(SKY-8965): Phase 2 — kill LLM synthesis, wire deterministic picker, flip guard to raise (#5596)

pedrohsdb0332d01e6e90
diff --git a/skyvern/core/script_generations/deterministic_field_naming.py b/skyvern/core/script_generations/deterministic_field_naming.py
index 197645efe..51e631f8b 100644
--- a/skyvern/core/script_generations/deterministic_field_naming.py
+++ b/skyvern/core/script_generations/deterministic_field_naming.py
@@ -147,28 +147,26 @@ def pick_field_name_for_action(
referenced_keys = extract_jinja_root_names(goal_template)
# Rule 1: jinja reference to a declared or schema key.
+ # Only fires when exactly ONE valid key is referenced in the goal — otherwise
+ # we can't disambiguate which INPUT_TEXT action targets which key and would
+ # collapse multiple fields onto the same name (CORR-1 from debate review).
intersection = referenced_keys & valid_keys
- if intersection:
- # When the goal references multiple keys we can't disambiguate here
- # (we don't know which INPUT_TEXT action targets which field). Prefer
- # a declared parameter over a schema key, then alphabetical for
- # determinism.
- # TODO(SKY-8965 Phase 2): before wiring this picker into the hot path,
- # disambiguate multi-key goals by falling through to Rule 3 (or a new
- # action-intention → key matcher) when `len(intersection) > 1`.
- # Otherwise every INPUT_TEXT action in a multi-field form gets the same
- # alphabetically-first name and the synthesized schema collapses fields.
- declared_matches = intersection & declared_param_keys
- preferred = sorted(declared_matches) if declared_matches else sorted(intersection)
- return FieldPick(field_name=preferred[0], rule="jinja_ref")
+ if len(intersection) == 1:
+ (match,) = intersection
+ return FieldPick(field_name=match, rule="jinja_ref")
# Rule 2: value matches an upstream schema key name or is clearly keyed
- # by intention to one. Conservative check: if the intention text mentions
- # a valid schema key (substring match, case-insensitive), bind to it.
+ # by intention to one. Sorted by descending key length so that `invoice_id`
+ # is tried before `id` — prevents short keys from shadowing longer,
+ # more-specific keys via substring match (andrewneilson review feedback).
+ # Keys that aren't valid Python identifiers are sanitized before returning
+ # (JSON Schema allows hyphens, reserved words, etc. that would produce
+ # invalid Pydantic class bodies).
intention = (action.get("intention") or "").lower()
- for key in sorted(upstream_schema_keys):
+ for key in sorted(upstream_schema_keys, key=len, reverse=True):
if key.lower() in intention:
- return FieldPick(field_name=key, rule="upstream_schema")
+ safe_key = sanitize_intention_to_field_name(key, fallback=key)
+ return FieldPick(field_name=safe_key, rule="upstream_schema")
# Rule 3: deterministic fallback derived from intention.
name = sanitize_intention_to_field_name(intention)
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.py
index 28681999b..d5e956568 100644
--- a/skyvern/core/script_generations/generate_script.py
+++ b/skyvern/core/script_generations/generate_script.py
@@ -120,6 +120,28 @@ def _collect_upstream_schema_keys(blocks: list[dict[str, Any]]) -> frozenset[str
return frozenset(keys)
+def _collect_goal_templates(blocks: list[dict[str, Any]]) -> dict[str, str]:
+ """Map each task_id to its unrendered navigation_goal template.
+
+ The deterministic picker uses these templates to detect jinja references
+ like ``{{ search_term }}`` and bind actions to declared parameters.
+
+ Recurses into ``loop_blocks`` so tasks nested inside ForLoops also get
+ Rule-1 context (CORR-3 from debate review).
+ """
+ result: dict[str, str] = {}
+ for block in blocks or []:
+ task_id = block.get("task_id")
+ if task_id:
+ goal = block.get("navigation_goal") or block.get("data_extraction_goal") or ""
+ if goal:
+ result[task_id] = goal
+ nested = block.get("loop_blocks")
+ if isinstance(nested, list):
+ result.update(_collect_goal_templates(nested))
+ return result
+
+
def _collect_synthesized_field_keys(schema_code: str) -> frozenset[str]:
"""Extract field names from a generated `GeneratedWorkflowParameters` class.
@@ -152,15 +174,26 @@ def _build_existing_field_assignments(
Returns:
Dictionary mapping action index (1-based) to the existing field name that must be preserved
"""
- # Build mapping of block label -> task_id
+ # Build mapping of block label -> task_id, recursing into loop_blocks so
+ # cached child blocks inside ForLoops are also covered. Without this,
+ # Phase 1-era LLM names in cached ForLoop children would not get
+ # re-registered and the Phase 2 guard would false-positive on regen
+ # (andrewneilson review feedback).
block_label_to_task_id: dict[str, str] = {}
- for idx, block in enumerate(blocks):
- if block.get("block_type") not in SCRIPT_TASK_BLOCKS:
- continue
- label = block.get("label") or block.get("title") or block.get("task_id") or f"task_{idx}"
- task_id = block.get("task_id")
- if task_id:
- block_label_to_task_id[label] = task_id
+
+ def _collect_block_label_to_task_id(blks: list[dict[str, Any]], counter: int = 0) -> None:
+ for block in blks:
+ if block.get("block_type") in SCRIPT_TASK_BLOCKS:
+ label = block.get("label") or block.get("title") or block.get("task_id") or f"task_{counter}"
+ task_id = block.get("task_id")
+ if task_id:
+ block_label_to_task_id[label] = task_id
+ nested = block.get("loop_blocks")
+ if isinstance(nested, list):
+ _collect_block_label_to_task_id(nested, counter + 1)
+ counter += 1
+
+ _collect_block_label_to_task_id(blocks)
# Build mapping of task_id -> list of existing field names (for unchanged blocks)
task_id_to_existing_fields: dict[str, list[str]] = {}
@@ -2877,8 +2910,18 @@ async def generate_workflow_script_python_code(
cached_blocks=cached_blocks,
updated_block_labels=updated_block_labels,
)
+ # SKY-8965 Phase 2: pass declared-param and upstream-schema keys so the
+ # deterministic picker can match actions to known fields instead of the
+ # LLM inventing names.
+ declared_keys = _collect_declared_param_keys(workflow)
+ upstream_keys = _collect_upstream_schema_keys(blocks)
+ goal_template_by_task = _collect_goal_templates(blocks)
generated_schema, field_mappings = await generate_workflow_parameters_schema(
- actions_by_task, existing_field_assignments
+ actions_by_task,
+ existing_field_assignments,
+ declared_param_keys=declared_keys,
+ upstream_schema_keys=upstream_keys,
+ goal_template_by_task=goal_template_by_task,
)
actions_by_task = hydrate_input_text_actions_with_field_names(actions_by_task, field_mappings)
@@ -3445,14 +3488,13 @@ async def generate_workflow_script_python_code(
module = cst.Module(body=module_body)
source_code = module.code
- # SKY-8965 Phase 1: validate `context.parameters[X]` references against the
+ # SKY-8965 Phase 2: validate `context.parameters[X]` references against the
# expanded valid-keys set (declared params ∪ upstream block schema keys ∪
- # synthesized `GeneratedWorkflowParameters` fields). Phase 1 logs violations
- # only — we want production telemetry before raising in Phase 2. Wrapped in
- # try/except so a guard bug can never break script generation.
+ # synthesized `GeneratedWorkflowParameters` fields). Phase 2 raises on
+ # violation, preventing phantom-parameter scripts from being cached.
+ # Phase 1 (now in prod) logged warnings; 13 violations in 7 days confirmed
+ # the bug is real and low-frequency (~0.05%), safe to enforce.
try:
- declared_keys = _collect_declared_param_keys(workflow)
- upstream_keys = _collect_upstream_schema_keys(blocks)
synthesized_keys = _collect_synthesized_field_keys(generated_schema)
guard_result = validate_context_parameter_refs(
code=source_code,
@@ -3462,12 +3504,23 @@ async def generate_workflow_script_python_code(
)
log_or_raise_guard_result(
guard_result,
- raise_on_violation=False,
+ raise_on_violation=True,
+ workflow_permanent_id=workflow.get("workflow_permanent_id"),
+ workflow_run_id=run_id,
+ )
+ except HallucinatedParameterError as hpe:
+ # Dedicated structured log so this is distinguishable from a codegen
+ # crash in Datadog. The broad except-Exception in the caller
+ # (workflow_script_service) would otherwise swallow the distinction
+ # (andrewneilson review feedback).
+ LOG.warning(
+ "parameter_reference_guard_rejected_script",
workflow_permanent_id=workflow.get("workflow_permanent_id"),
workflow_run_id=run_id,
+ undeclared_refs=sorted(r.key for r in hpe.result.undeclared_refs),
+ valid_keys=sorted(hpe.result.valid_keys),
+ sky_ticket="SKY-8965",
)
- except HallucinatedParameterError:
- # Phase 2 will raise this intentionally — don't swallow it.
raise
except Exception:
LOG.warning("parameter_reference_guard_failed_to_run", exc_info=True)
diff --git a/skyvern/core/script_generations/generate_workflow_parameters.py b/skyvern/core/script_generations/generate_workflow_parameters.py
index 7207fe672..862b325bb 100644
--- a/skyvern/core/script_generations/generate_workflow_parameters.py
+++ b/skyvern/core/script_generations/generate_workflow_parameters.py
@@ -1,62 +1,113 @@
"""
-Module for generating GeneratedWorkflowParameters schema from workflow run input_text actions.
+Module for generating GeneratedWorkflowParameters schema from workflow run
+input_text actions.
+
+SKY-8965 Phase 2: the LLM call (`_generate_field_names_with_llm`) has been
+replaced by a deterministic 3-rule picker. See
+`deterministic_field_naming.py` for the picker and
+`parameter_reference_guard.py` for the post-generation validation guard.
"""
from typing import Any, Dict, List, Tuple
import structlog
-from pydantic import BaseModel
-from skyvern.forge import app
+from skyvern.core.script_generations.deterministic_field_naming import (
+ pick_field_names_for_actions,
+)
from skyvern.forge.sdk.core import skyvern_context
-from skyvern.forge.sdk.prompting import PromptEngine
from skyvern.webeye.actions.actions import ActionType
LOG = structlog.get_logger(__name__)
-# Initialize prompt engine
-prompt_engine = PromptEngine("skyvern")
CUSTOM_FIELD_ACTIONS = [ActionType.INPUT_TEXT, ActionType.UPLOAD_FILE, ActionType.SELECT_OPTION]
-class GeneratedFieldMapping(BaseModel):
- """Mapping of action indices to field names."""
-
- field_mappings: Dict[str, str]
- schema_fields: Dict[str, Dict[str, str]]
-
-
+# async retained for caller-signature compat; no I/O inside after Phase 2
+# removed the LLM call.
async def generate_workflow_parameters_schema(
actions_by_task: Dict[str, List[Dict[str, Any]]],
existing_field_assignments: Dict[int, str] | None = None,
+ *,
+ declared_param_keys: frozenset[str] = frozenset(),
+ upstream_schema_keys: frozenset[str] = frozenset(),
+ goal_template_by_task: Dict[str, str] | None = None,
) -> Tuple[str, Dict[str, str]]:
"""
Generate a GeneratedWorkflowParameters Pydantic schema based on input_text actions.
- Args:
- actions_by_task: Dictionary mapping task IDs to lists of action dictionaries
- existing_field_assignments: Optional dictionary mapping action index (1-based) to
- existing field names that must be preserved. Used when regenerating schemas
- to maintain compatibility with cached block code.
+ SKY-8965 Phase 2: field names are picked deterministically (no LLM call).
+ The three-rule picker in ``deterministic_field_naming.py`` decides each
+ name based on: (1) jinja refs in the goal, (2) upstream schema keys,
+ (3) sanitized action intention.
+
+ New keyword-only args (optional for backwards compat — callers that don't
+ pass them get empty frozensets, which makes the picker fall through to
+ Rule 3 for every action):
+ declared_param_keys: Workflow-declared parameter keys.
+ upstream_schema_keys: Keys from upstream blocks' extraction schemas.
+ goal_template_by_task: Unrendered navigation_goal per task_id.
Returns:
Tuple of (schema_code, field_mappings) where:
- schema_code: Python code for the GeneratedWorkflowParameters class
- - field_mappings: Dictionary mapping action indices to field names for hydration
+ - field_mappings: Dictionary mapping ``"{task_id}:{action_id}"`` to field
+ names, used by ``hydrate_input_text_actions_with_field_names``
"""
- existing_field_assignments = existing_field_assignments or {}
+ goal_template_by_task = goal_template_by_task or {}
+
+ # --- mark incomplete actions (SKY-7653 race-condition mitigation) ----
+ _mark_incomplete_actions(actions_by_task)
+
+ # --- deterministic pick (replaces the LLM call) ---------------------
+ picks = pick_field_names_for_actions(
+ actions_by_task=actions_by_task,
+ goal_template_by_task=goal_template_by_task,
+ declared_param_keys=declared_param_keys,
+ upstream_schema_keys=upstream_schema_keys,
+ existing_field_assignments=existing_field_assignments,
+ )
+
+ if not picks:
+ LOG.info("No custom-field actions found — empty schema")
+ return _generate_empty_schema(), {}
+
+ # Build schema and field mappings from the picks
+ schema_fields: Dict[str, Dict[str, str]] = {}
+ action_field_mappings: Dict[str, str] = {}
- # Extract all input_text actions
- custom_field_actions = []
- action_index_map = {}
- action_counter = 1
+ for action_key, pick in picks.items():
+ action_field_mappings[action_key] = pick.field_name
+ if pick.field_name not in schema_fields:
+ schema_fields[pick.field_name] = {
+ "type": "str",
+ "description": pick.description or f"Value for {pick.field_name}",
+ }
+
+ LOG.info(
+ "deterministic_field_naming_complete",
+ total_picks=len(picks),
+ rules_used=sorted({pick.rule for pick in picks.values()}),
+ field_names=sorted(schema_fields.keys()),
+ )
+
+ schema_code = _generate_pydantic_schema(schema_fields)
+ return schema_code, action_field_mappings
+
+
+def _mark_incomplete_actions(actions_by_task: Dict[str, List[Dict[str, Any]]]) -> None:
+ """Flag the skyvern context if any custom-field actions lack data (SKY-7653).
+ This preserves the race-condition mitigation from the original LLM path:
+ when script generation runs before action data is fully saved to the DB,
+ the ``script_gen_had_incomplete_actions`` flag triggers a finalize
+ regeneration.
+ """
for task_id, actions in actions_by_task.items():
for action in actions:
action_type = action.get("action_type", "")
if action_type not in CUSTOM_FIELD_ACTIONS:
continue
-
value = ""
if action_type == ActionType.INPUT_TEXT:
value = action.get("text", "")
@@ -64,93 +115,11 @@ async def generate_workflow_parameters_schema(
value = action.get("file_url", "")
elif action_type == ActionType.SELECT_OPTION:
value = action.get("option", "")
-
- # Skip actions without data - addresses race condition where script generation
- # runs before action data is fully saved to database (SKY-7653)
if not value:
- LOG.debug(
- "Skipping action without data during field mapping generation",
- action_type=action_type,
- action_id=action.get("action_id", ""),
- task_id=task_id,
- )
- # Mark that we had incomplete actions - triggers finalize regeneration
ctx = skyvern_context.current()
if ctx:
ctx.script_gen_had_incomplete_actions = True
- continue
-
- # Check if this action has an existing field name that must be preserved
- existing_field_name = existing_field_assignments.get(action_counter)
-
- custom_field_actions.append(
- {
- "action_type": action_type,
- "value": value,
- "intention": action.get("intention", ""),
- "task_id": task_id,
- "action_id": action.get("action_id", ""),
- "existing_field_name": existing_field_name,
- }
- )
- action_index_map[f"action_index_{action_counter}"] = {
- "task_id": task_id,
- "action_id": action.get("action_id", ""),
- }
- action_counter += 1
-
- if not custom_field_actions:
- LOG.warning("No field_name_actions found in workflow run")
- return _generate_empty_schema(), {}
-
- # Generate field names using LLM
- try:
- field_mapping = await _generate_field_names_with_llm(custom_field_actions)
-
- # Generate the Pydantic schema code
- schema_code = _generate_pydantic_schema(field_mapping.schema_fields)
-
- # Create field mappings for action hydration
- action_field_mappings = {}
- for action_idx, field_name in field_mapping.field_mappings.items():
- if action_idx in action_index_map:
- action_info = action_index_map[action_idx]
- key = f"{action_info['task_id']}:{action_info['action_id']}"
- action_field_mappings[key] = field_name
-
- return schema_code, action_field_mappings
-
- except Exception as e:
- LOG.error("Failed to generate workflow parameters schema", error=str(e), exc_info=True)
- return _generate_empty_schema(), {}
-
-
-async def _generate_field_names_with_llm(
- custom_field_actions: List[Dict[str, Any]],
-) -> GeneratedFieldMapping:
- """
- Use LLM to generate field names from input actions.
-
- Args:
- custom_field_actions: List of action dictionaries with action details.
- Each action may include an "existing_field_name" key if the field
- name must be preserved from a cached block.
-
- Returns:
- GeneratedFieldMapping with field mappings and schema definitions
- """
- # Check if any actions have existing field names that must be preserved
- has_existing_fields = any(action.get("existing_field_name") for action in custom_field_actions)
-
- prompt = prompt_engine.load_prompt(
- template="generate-workflow-parameters",
- custom_field_actions=custom_field_actions,
- has_existing_fields=has_existing_fields,
- )
-
- response = await app.SCRIPT_GENERATION_LLM_API_HANDLER(prompt=prompt, prompt_name="generate-workflow-parameters")
-
- return GeneratedFieldMapping.model_validate(response)
+ return # one flag is enough
def _generate_pydantic_schema(schema_fields: Dict[str, Dict[str, str]]) -> str:
diff --git a/tests/unit/test_deterministic_field_naming.py b/tests/unit/test_deterministic_field_naming.py
index c41efe671..ac378ca00 100644
--- a/tests/unit/test_deterministic_field_naming.py
+++ b/tests/unit/test_deterministic_field_naming.py
@@ -100,16 +100,18 @@ def test_rule_1_jinja_reference_to_declared_param() -> None:
assert pick.rule == "jinja_ref"
-def test_rule_1_prefers_declared_param_over_schema_key_on_tie() -> None:
- """When both sets match, declared-param beats schema-key."""
+def test_rule_1_falls_through_on_multi_key_goal() -> None:
+ """When the goal references multiple valid keys, Rule 1 can't disambiguate
+ which INPUT_TEXT action targets which key. Fall through to Rule 3 to avoid
+ collapsing all actions onto the same name (CORR-1 from debate review)."""
pick = pick_field_name_for_action(
action=_input_action("x", intention="pick one"),
goal_template="use {{ name_declared }} and {{ name_schema }}",
declared_param_keys=frozenset({"name_declared"}),
upstream_schema_keys=frozenset({"name_schema"}),
)
- assert pick.field_name == "name_declared"
- assert pick.rule == "jinja_ref"
+ # Multi-key → Rule 3 fallback, not Rule 1
+ assert pick.rule == "intention_derived"
def test_rule_1_ignores_jinja_refs_not_in_valid_keys() -> None:
diff --git a/tests/unit/test_script_generation_race_condition.py b/tests/unit/test_script_generation_race_condition.py
index 44a783f21..859debf7e 100644
--- a/tests/unit/test_script_generation_race_condition.py
+++ b/tests/unit/test_script_generation_race_condition.py
@@ -12,10 +12,8 @@ from typing import Any
import pytest
-from skyvern.core.script_generations import generate_workflow_parameters as gwp
from skyvern.core.script_generations.generate_workflow_parameters import (
CUSTOM_FIELD_ACTIONS,
- GeneratedFieldMapping,
generate_workflow_parameters_schema,
hydrate_input_text_actions_with_field_names,
)
@@ -200,16 +198,6 @@ class TestCodeGenerationWithoutFieldName:
class TestFieldMappingGeneration:
"""Test the field mapping generation logic."""
- def test_field_mapping_structure(self) -> None:
- """Test that GeneratedFieldMapping has the expected structure."""
- mapping = GeneratedFieldMapping(
- field_mappings={"action_index_1": "facility_name"},
- schema_fields={"facility_name": {"type": "str", "description": "The facility name"}},
- )
-
- assert mapping.field_mappings["action_index_1"] == "facility_name"
- assert mapping.schema_fields["facility_name"]["type"] == "str"
-
def test_action_index_to_field_mapping_key_format(self) -> None:
"""Test that field mapping keys use the correct format: task_id:action_id."""
task_id = "task-123"
@@ -243,21 +231,9 @@ async def test_generate_workflow_parameters_schema_empty_actions(monkeypatch: py
@pytest.mark.asyncio
async def test_generate_workflow_parameters_schema_with_actions(monkeypatch: pytest.MonkeyPatch) -> None:
"""
- Integration test: Verify that when actions are present, LLM is called.
-
- This confirms that when script generation runs AFTER actions are saved,
- it properly generates field mappings.
+ Integration test: Verify that when actions are present, the deterministic
+ picker generates field mappings (SKY-8965 Phase 2: no LLM call).
"""
-
- # Mock the LLM call to return a mapping
- async def mock_generate_field_names_with_llm(custom_field_actions):
- return GeneratedFieldMapping(
- field_mappings={"action_index_1": "facility_name"},
- schema_fields={"facility_name": {"type": "str", "description": "The facility name"}},
- )
-
- monkeypatch.setattr(gwp, "_generate_field_names_with_llm", mock_generate_field_names_with_llm)
-
task_id = "task-123"
action_id = "action-456"
actions_by_task = {
@@ -268,13 +244,13 @@ async def test_generate_workflow_parameters_schema_with_actions(monkeypatch: pyt
schema_code, action_field_mappings = await generate_workflow_parameters_schema(actions_by_task)
- # Should have generated schema with field
- assert "facility_name" in schema_code
+ # Should have generated schema with a field derived from the intention
assert "GeneratedWorkflowParameters" in schema_code
+ assert "enter_facility_name" in schema_code
# Should have mapping for our action
assert f"{task_id}:{action_id}" in action_field_mappings
- assert action_field_mappings[f"{task_id}:{action_id}"] == "facility_name"
+ assert action_field_mappings[f"{task_id}:{action_id}"] == "enter_facility_name"
class TestRaceConditionTimingScenario:
@@ -662,19 +638,6 @@ async def test_generate_workflow_parameters_schema_skips_empty_actions_and_sets_
context = SkyvernContext()
skyvern_context.set(context)
- # Mock the LLM call - should only be called if there are valid actions
- llm_called = False
-
- async def mock_generate_field_names_with_llm(custom_field_actions):
- nonlocal llm_called
- llm_called = True
- return GeneratedFieldMapping(
- field_mappings={"action_index_1": "facility_name"},
- schema_fields={"facility_name": {"type": "str", "description": "The facility name"}},
- )
-
- monkeypatch.setattr(gwp, "_generate_field_names_with_llm", mock_generate_field_names_with_llm)
-
task_id = "task-123"
# Actions with empty values - simulates race condition
@@ -693,9 +656,7 @@ async def test_generate_workflow_parameters_schema_skips_empty_actions_and_sets_
try:
schema_code, action_field_mappings = await generate_workflow_parameters_schema(actions_by_task)
- # LLM should NOT be called because action was skipped
- assert not llm_called
-
+ # Deterministic picker skips empty-value actions (same as old LLM path)
# Should return empty schema
assert "pass" in schema_code
assert action_field_mappings == {}
@@ -720,15 +681,6 @@ async def test_generate_workflow_parameters_schema_with_complete_actions_no_flag
context = SkyvernContext()
skyvern_context.set(context)
- # Mock the LLM call
- async def mock_generate_field_names_with_llm(custom_field_actions):
- return GeneratedFieldMapping(
- field_mappings={"action_index_1": "facility_name"},
- schema_fields={"facility_name": {"type": "str", "description": "The facility name"}},
- )
-
- monkeypatch.setattr(gwp, "_generate_field_names_with_llm", mock_generate_field_names_with_llm)
-
task_id = "task-123"
# Actions with complete values - no race condition
@@ -747,8 +699,8 @@ async def test_generate_workflow_parameters_schema_with_complete_actions_no_flag
try:
schema_code, action_field_mappings = await generate_workflow_parameters_schema(actions_by_task)
- # Should have generated schema
- assert "facility_name" in schema_code
+ # Should have generated schema with deterministic field name
+ assert "enter_facility_name" in schema_code
# Context flag should NOT be set - no regeneration needed
assert context.script_gen_had_incomplete_actions is False
diff --git a/tests/unit/test_workflow_schema_field_preservation.py b/tests/unit/test_workflow_schema_field_preservation.py
index 3f3dbe49e..c1580d4db 100644
--- a/tests/unit/test_workflow_schema_field_preservation.py
+++ b/tests/unit/test_workflow_schema_field_preservation.py
@@ -231,43 +231,18 @@ class TestGenerateWorkflowParametersSchemaWithExistingFields:
# Action 3 has no existing field - needs new name
}
- # Mock the LLM response
- mock_llm_response = {
- "field_mappings": {
- "action_index_1": "preserved_username",
- "action_index_2": "preserved_password",
- "action_index_3": "new_field_name",
- },
- "schema_fields": {
- "preserved_username": {"type": "str", "description": "Username"},
- "preserved_password": {"type": "str", "description": "Password"},
- "new_field_name": {"type": "str", "description": "New field"},
- },
- }
-
- captured_prompt = {}
-
- async def mock_llm_handler(prompt, prompt_name):
- captured_prompt["prompt"] = prompt
- captured_prompt["prompt_name"] = prompt_name
- return mock_llm_response
-
- self.stub_app.SCRIPT_GENERATION_LLM_API_HANDLER = AsyncMock(side_effect=mock_llm_handler)
-
+ # SKY-8965 Phase 2: no LLM mock needed — deterministic picker handles
+ # field-name assignment. Existing field assignments are preserved by the
+ # picker's "existing_assignment" rule (highest priority).
schema_code, field_mappings = await generate_workflow_parameters_schema(
actions_by_task, existing_field_assignments
)
- # Verify the prompt contains the existing field names
- prompt = captured_prompt["prompt"]
- assert "preserved_username" in prompt
- assert "preserved_password" in prompt
- assert "MUST PRESERVE" in prompt or "EXISTING FIELD NAME" in prompt
-
- # Verify the returned field mappings include preserved names
+ # Verify existing field names are preserved by the deterministic picker
assert field_mappings["task_1:act_1"] == "preserved_username"
assert field_mappings["task_1:act_2"] == "preserved_password"
- assert field_mappings["task_2:act_3"] == "new_field_name"
+ # New action gets a deterministic name from its intention ("Enter new field")
+ assert field_mappings["task_2:act_3"] == "enter_new_field"
@pytest.mark.asyncio
async def test_no_existing_fields_works_normally(self):
@@ -279,34 +254,15 @@ class TestGenerateWorkflowParametersSchemaWithExistingFields:
}
existing_field_assignments: dict[int, str] = {} # No existing fields
- mock_llm_response = {
- "field_mappings": {
- "action_index_1": "username",
- },
- "schema_fields": {
- "username": {"type": "str", "description": "Username field"},
- },
- }
-
- captured_prompt = {}
-
- async def mock_llm_handler(prompt, prompt_name):
- captured_prompt["prompt"] = prompt
- return mock_llm_response
-
- self.stub_app.SCRIPT_GENERATION_LLM_API_HANDLER = AsyncMock(side_effect=mock_llm_handler)
-
+ # SKY-8965 Phase 2: no LLM call, no prompt to inspect. Deterministic
+ # picker falls through to Rule 3 (intention-derived).
schema_code, field_mappings = await generate_workflow_parameters_schema(
actions_by_task, existing_field_assignments
)
- # Should not contain preservation instructions when no existing fields
- prompt = captured_prompt["prompt"]
- # The CRITICAL rule only appears when has_existing_fields is True
- assert "CRITICAL" not in prompt
-
- # Should still return valid mappings
- assert field_mappings["task_1:act_1"] == "username"
+ # Should return valid mappings with deterministic name from intention
+ assert "task_1:act_1" in field_mappings
+ assert field_mappings["task_1:act_1"] == "enter_username"
@pytest.mark.asyncio
async def test_schema_code_includes_preserved_field_names(self):
@@ -427,47 +383,23 @@ async def login_block(page: SkyvernPage, context: RunContext):
# Action 3 has no existing field (new block)
}
- # Step 2: Mock LLM that respects the preservation instructions
- mock_llm_response = {
- "field_mappings": {
- "action_index_1": "user_email", # Preserved
- "action_index_2": "user_password", # Preserved
- "action_index_3": "company_name", # New field for new block
- },
- "schema_fields": {
- "user_email": {"type": "str", "description": "User's email address"},
- "user_password": {"type": "str", "description": "User's password"},
- "company_name": {"type": "str", "description": "Company name"},
- },
- }
-
- captured_prompt = {}
-
- async def mock_llm_handler(prompt, prompt_name):
- captured_prompt["prompt"] = prompt
- return mock_llm_response
-
- self.stub_app.SCRIPT_GENERATION_LLM_API_HANDLER = AsyncMock(side_effect=mock_llm_handler)
-
+ # Step 2: SKY-8965 Phase 2 — deterministic picker preserves existing
+ # field names and assigns new ones from intention.
schema_code, field_mappings = await generate_workflow_parameters_schema(
actions_by_task, existing_field_assignments
)
- # Verify the prompt contains preservation instructions
- prompt = captured_prompt["prompt"]
- assert "user_email" in prompt, "Prompt should contain existing field name 'user_email'"
- assert "user_password" in prompt, "Prompt should contain existing field name 'user_password'"
- assert "MUST PRESERVE" in prompt or "EXISTING FIELD NAME" in prompt
-
- # Verify field mappings preserve the original names
+ # Verify field mappings preserve the original names via "existing_assignment" rule
assert field_mappings["task_1:act_1"] == "user_email", "Login block email field should be preserved"
assert field_mappings["task_1:act_2"] == "user_password", "Login block password field should be preserved"
- assert field_mappings["task_2:act_3"] == "company_name", "New block should get new field name"
# Verify schema code contains preserved field names
assert "user_email" in schema_code
assert "user_password" in schema_code
- assert "company_name" in schema_code
+
+ # New block gets a deterministic name from its intention ("Enter company name")
+ assert field_mappings["task_2:act_3"] == "enter_company_name"
+ assert "enter_company_name" in schema_code
# The cached login block code references context.parameters['user_email']
# and context.parameters['user_password'], which now match the schema!