Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-8965): Phase 2 — kill LLM synthesis, wire deterministic picker, flip guard to raise (#5596)
pedrohsdb0332d01e6e90
diff --git a/skyvern/core/script_generations/deterministic_field_naming.py b/skyvern/core/script_generations/deterministic_field_naming.pyindex 197645efe..51e631f8b 100644--- a/skyvern/core/script_generations/deterministic_field_naming.py+++ b/skyvern/core/script_generations/deterministic_field_naming.py@@ -147,28 +147,26 @@ def pick_field_name_for_action(referenced_keys = extract_jinja_root_names(goal_template)# Rule 1: jinja reference to a declared or schema key.+ # Only fires when exactly ONE valid key is referenced in the goal — otherwise+ # we can't disambiguate which INPUT_TEXT action targets which key and would+ # collapse multiple fields onto the same name (CORR-1 from debate review).intersection = referenced_keys & valid_keys- if intersection:- # When the goal references multiple keys we can't disambiguate here- # (we don't know which INPUT_TEXT action targets which field). Prefer- # a declared parameter over a schema key, then alphabetical for- # determinism.- # TODO(SKY-8965 Phase 2): before wiring this picker into the hot path,- # disambiguate multi-key goals by falling through to Rule 3 (or a new- # action-intention → key matcher) when `len(intersection) > 1`.- # Otherwise every INPUT_TEXT action in a multi-field form gets the same- # alphabetically-first name and the synthesized schema collapses fields.- declared_matches = intersection & declared_param_keys- preferred = sorted(declared_matches) if declared_matches else sorted(intersection)- return FieldPick(field_name=preferred[0], rule="jinja_ref")+ if len(intersection) == 1:+ (match,) = intersection+ return FieldPick(field_name=match, rule="jinja_ref")# Rule 2: value matches an upstream schema key name or is clearly keyed- # by intention to one. Conservative check: if the intention text mentions- # a valid schema key (substring match, case-insensitive), bind to it.+ # by intention to one. Sorted by descending key length so that `invoice_id`+ # is tried before `id` — prevents short keys from shadowing longer,+ # more-specific keys via substring match (andrewneilson review feedback).+ # Keys that aren't valid Python identifiers are sanitized before returning+ # (JSON Schema allows hyphens, reserved words, etc. that would produce+ # invalid Pydantic class bodies).intention = (action.get("intention") or "").lower()- for key in sorted(upstream_schema_keys):+ for key in sorted(upstream_schema_keys, key=len, reverse=True):if key.lower() in intention:- return FieldPick(field_name=key, rule="upstream_schema")+ safe_key = sanitize_intention_to_field_name(key, fallback=key)+ return FieldPick(field_name=safe_key, rule="upstream_schema")# Rule 3: deterministic fallback derived from intention.name = sanitize_intention_to_field_name(intention)diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.pyindex 28681999b..d5e956568 100644--- a/skyvern/core/script_generations/generate_script.py+++ b/skyvern/core/script_generations/generate_script.py@@ -120,6 +120,28 @@ def _collect_upstream_schema_keys(blocks: list[dict[str, Any]]) -> frozenset[strreturn frozenset(keys)+def _collect_goal_templates(blocks: list[dict[str, Any]]) -> dict[str, str]:+ """Map each task_id to its unrendered navigation_goal template.++ The deterministic picker uses these templates to detect jinja references+ like ``{{ search_term }}`` and bind actions to declared parameters.++ Recurses into ``loop_blocks`` so tasks nested inside ForLoops also get+ Rule-1 context (CORR-3 from debate review).+ """+ result: dict[str, str] = {}+ for block in blocks or []:+ task_id = block.get("task_id")+ if task_id:+ goal = block.get("navigation_goal") or block.get("data_extraction_goal") or ""+ if goal:+ result[task_id] = goal+ nested = block.get("loop_blocks")+ if isinstance(nested, list):+ result.update(_collect_goal_templates(nested))+ return result++def _collect_synthesized_field_keys(schema_code: str) -> frozenset[str]:"""Extract field names from a generated `GeneratedWorkflowParameters` class.@@ -152,15 +174,26 @@ def _build_existing_field_assignments(Returns:Dictionary mapping action index (1-based) to the existing field name that must be preserved"""- # Build mapping of block label -> task_id+ # Build mapping of block label -> task_id, recursing into loop_blocks so+ # cached child blocks inside ForLoops are also covered. Without this,+ # Phase 1-era LLM names in cached ForLoop children would not get+ # re-registered and the Phase 2 guard would false-positive on regen+ # (andrewneilson review feedback).block_label_to_task_id: dict[str, str] = {}- for idx, block in enumerate(blocks):- if block.get("block_type") not in SCRIPT_TASK_BLOCKS:- continue- label = block.get("label") or block.get("title") or block.get("task_id") or f"task_{idx}"- task_id = block.get("task_id")- if task_id:- block_label_to_task_id[label] = task_id++ def _collect_block_label_to_task_id(blks: list[dict[str, Any]], counter: int = 0) -> None:+ for block in blks:+ if block.get("block_type") in SCRIPT_TASK_BLOCKS:+ label = block.get("label") or block.get("title") or block.get("task_id") or f"task_{counter}"+ task_id = block.get("task_id")+ if task_id:+ block_label_to_task_id[label] = task_id+ nested = block.get("loop_blocks")+ if isinstance(nested, list):+ _collect_block_label_to_task_id(nested, counter + 1)+ counter += 1++ _collect_block_label_to_task_id(blocks)# Build mapping of task_id -> list of existing field names (for unchanged blocks)task_id_to_existing_fields: dict[str, list[str]] = {}@@ -2877,8 +2910,18 @@ async def generate_workflow_script_python_code(cached_blocks=cached_blocks,updated_block_labels=updated_block_labels,)+ # SKY-8965 Phase 2: pass declared-param and upstream-schema keys so the+ # deterministic picker can match actions to known fields instead of the+ # LLM inventing names.+ declared_keys = _collect_declared_param_keys(workflow)+ upstream_keys = _collect_upstream_schema_keys(blocks)+ goal_template_by_task = _collect_goal_templates(blocks)generated_schema, field_mappings = await generate_workflow_parameters_schema(- actions_by_task, existing_field_assignments+ actions_by_task,+ existing_field_assignments,+ declared_param_keys=declared_keys,+ upstream_schema_keys=upstream_keys,+ goal_template_by_task=goal_template_by_task,)actions_by_task = hydrate_input_text_actions_with_field_names(actions_by_task, field_mappings)@@ -3445,14 +3488,13 @@ async def generate_workflow_script_python_code(module = cst.Module(body=module_body)source_code = module.code- # SKY-8965 Phase 1: validate `context.parameters[X]` references against the+ # SKY-8965 Phase 2: validate `context.parameters[X]` references against the# expanded valid-keys set (declared params ∪ upstream block schema keys ∪- # synthesized `GeneratedWorkflowParameters` fields). Phase 1 logs violations- # only — we want production telemetry before raising in Phase 2. Wrapped in- # try/except so a guard bug can never break script generation.+ # synthesized `GeneratedWorkflowParameters` fields). Phase 2 raises on+ # violation, preventing phantom-parameter scripts from being cached.+ # Phase 1 (now in prod) logged warnings; 13 violations in 7 days confirmed+ # the bug is real and low-frequency (~0.05%), safe to enforce.try:- declared_keys = _collect_declared_param_keys(workflow)- upstream_keys = _collect_upstream_schema_keys(blocks)synthesized_keys = _collect_synthesized_field_keys(generated_schema)guard_result = validate_context_parameter_refs(code=source_code,@@ -3462,12 +3504,23 @@ async def generate_workflow_script_python_code()log_or_raise_guard_result(guard_result,- raise_on_violation=False,+ raise_on_violation=True,+ workflow_permanent_id=workflow.get("workflow_permanent_id"),+ workflow_run_id=run_id,+ )+ except HallucinatedParameterError as hpe:+ # Dedicated structured log so this is distinguishable from a codegen+ # crash in Datadog. The broad except-Exception in the caller+ # (workflow_script_service) would otherwise swallow the distinction+ # (andrewneilson review feedback).+ LOG.warning(+ "parameter_reference_guard_rejected_script",workflow_permanent_id=workflow.get("workflow_permanent_id"),workflow_run_id=run_id,+ undeclared_refs=sorted(r.key for r in hpe.result.undeclared_refs),+ valid_keys=sorted(hpe.result.valid_keys),+ sky_ticket="SKY-8965",)- except HallucinatedParameterError:- # Phase 2 will raise this intentionally — don't swallow it.raiseexcept Exception:LOG.warning("parameter_reference_guard_failed_to_run", exc_info=True)diff --git a/skyvern/core/script_generations/generate_workflow_parameters.py b/skyvern/core/script_generations/generate_workflow_parameters.pyindex 7207fe672..862b325bb 100644--- a/skyvern/core/script_generations/generate_workflow_parameters.py+++ b/skyvern/core/script_generations/generate_workflow_parameters.py@@ -1,62 +1,113 @@"""-Module for generating GeneratedWorkflowParameters schema from workflow run input_text actions.+Module for generating GeneratedWorkflowParameters schema from workflow run+input_text actions.++SKY-8965 Phase 2: the LLM call (`_generate_field_names_with_llm`) has been+replaced by a deterministic 3-rule picker. See+`deterministic_field_naming.py` for the picker and+`parameter_reference_guard.py` for the post-generation validation guard."""from typing import Any, Dict, List, Tupleimport structlog-from pydantic import BaseModel-from skyvern.forge import app+from skyvern.core.script_generations.deterministic_field_naming import (+ pick_field_names_for_actions,+)from skyvern.forge.sdk.core import skyvern_context-from skyvern.forge.sdk.prompting import PromptEnginefrom skyvern.webeye.actions.actions import ActionTypeLOG = structlog.get_logger(__name__)-# Initialize prompt engine-prompt_engine = PromptEngine("skyvern")CUSTOM_FIELD_ACTIONS = [ActionType.INPUT_TEXT, ActionType.UPLOAD_FILE, ActionType.SELECT_OPTION]-class GeneratedFieldMapping(BaseModel):- """Mapping of action indices to field names."""-- field_mappings: Dict[str, str]- schema_fields: Dict[str, Dict[str, str]]--+# async retained for caller-signature compat; no I/O inside after Phase 2+# removed the LLM call.async def generate_workflow_parameters_schema(actions_by_task: Dict[str, List[Dict[str, Any]]],existing_field_assignments: Dict[int, str] | None = None,+ *,+ declared_param_keys: frozenset[str] = frozenset(),+ upstream_schema_keys: frozenset[str] = frozenset(),+ goal_template_by_task: Dict[str, str] | None = None,) -> Tuple[str, Dict[str, str]]:"""Generate a GeneratedWorkflowParameters Pydantic schema based on input_text actions.- Args:- actions_by_task: Dictionary mapping task IDs to lists of action dictionaries- existing_field_assignments: Optional dictionary mapping action index (1-based) to- existing field names that must be preserved. Used when regenerating schemas- to maintain compatibility with cached block code.+ SKY-8965 Phase 2: field names are picked deterministically (no LLM call).+ The three-rule picker in ``deterministic_field_naming.py`` decides each+ name based on: (1) jinja refs in the goal, (2) upstream schema keys,+ (3) sanitized action intention.++ New keyword-only args (optional for backwards compat — callers that don't+ pass them get empty frozensets, which makes the picker fall through to+ Rule 3 for every action):+ declared_param_keys: Workflow-declared parameter keys.+ upstream_schema_keys: Keys from upstream blocks' extraction schemas.+ goal_template_by_task: Unrendered navigation_goal per task_id.Returns:Tuple of (schema_code, field_mappings) where:- schema_code: Python code for the GeneratedWorkflowParameters class- - field_mappings: Dictionary mapping action indices to field names for hydration+ - field_mappings: Dictionary mapping ``"{task_id}:{action_id}"`` to field+ names, used by ``hydrate_input_text_actions_with_field_names``"""- existing_field_assignments = existing_field_assignments or {}+ goal_template_by_task = goal_template_by_task or {}++ # --- mark incomplete actions (SKY-7653 race-condition mitigation) ----+ _mark_incomplete_actions(actions_by_task)++ # --- deterministic pick (replaces the LLM call) ---------------------+ picks = pick_field_names_for_actions(+ actions_by_task=actions_by_task,+ goal_template_by_task=goal_template_by_task,+ declared_param_keys=declared_param_keys,+ upstream_schema_keys=upstream_schema_keys,+ existing_field_assignments=existing_field_assignments,+ )++ if not picks:+ LOG.info("No custom-field actions found — empty schema")+ return _generate_empty_schema(), {}++ # Build schema and field mappings from the picks+ schema_fields: Dict[str, Dict[str, str]] = {}+ action_field_mappings: Dict[str, str] = {}- # Extract all input_text actions- custom_field_actions = []- action_index_map = {}- action_counter = 1+ for action_key, pick in picks.items():+ action_field_mappings[action_key] = pick.field_name+ if pick.field_name not in schema_fields:+ schema_fields[pick.field_name] = {+ "type": "str",+ "description": pick.description or f"Value for {pick.field_name}",+ }++ LOG.info(+ "deterministic_field_naming_complete",+ total_picks=len(picks),+ rules_used=sorted({pick.rule for pick in picks.values()}),+ field_names=sorted(schema_fields.keys()),+ )++ schema_code = _generate_pydantic_schema(schema_fields)+ return schema_code, action_field_mappings+++def _mark_incomplete_actions(actions_by_task: Dict[str, List[Dict[str, Any]]]) -> None:+ """Flag the skyvern context if any custom-field actions lack data (SKY-7653).+ This preserves the race-condition mitigation from the original LLM path:+ when script generation runs before action data is fully saved to the DB,+ the ``script_gen_had_incomplete_actions`` flag triggers a finalize+ regeneration.+ """for task_id, actions in actions_by_task.items():for action in actions:action_type = action.get("action_type", "")if action_type not in CUSTOM_FIELD_ACTIONS:continue-value = ""if action_type == ActionType.INPUT_TEXT:value = action.get("text", "")@@ -64,93 +115,11 @@ async def generate_workflow_parameters_schema(value = action.get("file_url", "")elif action_type == ActionType.SELECT_OPTION:value = action.get("option", "")-- # Skip actions without data - addresses race condition where script generation- # runs before action data is fully saved to database (SKY-7653)if not value:- LOG.debug(- "Skipping action without data during field mapping generation",- action_type=action_type,- action_id=action.get("action_id", ""),- task_id=task_id,- )- # Mark that we had incomplete actions - triggers finalize regenerationctx = skyvern_context.current()if ctx:ctx.script_gen_had_incomplete_actions = True- continue-- # Check if this action has an existing field name that must be preserved- existing_field_name = existing_field_assignments.get(action_counter)-- custom_field_actions.append(- {- "action_type": action_type,- "value": value,- "intention": action.get("intention", ""),- "task_id": task_id,- "action_id": action.get("action_id", ""),- "existing_field_name": existing_field_name,- }- )- action_index_map[f"action_index_{action_counter}"] = {- "task_id": task_id,- "action_id": action.get("action_id", ""),- }- action_counter += 1-- if not custom_field_actions:- LOG.warning("No field_name_actions found in workflow run")- return _generate_empty_schema(), {}-- # Generate field names using LLM- try:- field_mapping = await _generate_field_names_with_llm(custom_field_actions)-- # Generate the Pydantic schema code- schema_code = _generate_pydantic_schema(field_mapping.schema_fields)-- # Create field mappings for action hydration- action_field_mappings = {}- for action_idx, field_name in field_mapping.field_mappings.items():- if action_idx in action_index_map:- action_info = action_index_map[action_idx]- key = f"{action_info['task_id']}:{action_info['action_id']}"- action_field_mappings[key] = field_name-- return schema_code, action_field_mappings-- except Exception as e:- LOG.error("Failed to generate workflow parameters schema", error=str(e), exc_info=True)- return _generate_empty_schema(), {}---async def _generate_field_names_with_llm(- custom_field_actions: List[Dict[str, Any]],-) -> GeneratedFieldMapping:- """- Use LLM to generate field names from input actions.-- Args:- custom_field_actions: List of action dictionaries with action details.- Each action may include an "existing_field_name" key if the field- name must be preserved from a cached block.-- Returns:- GeneratedFieldMapping with field mappings and schema definitions- """- # Check if any actions have existing field names that must be preserved- has_existing_fields = any(action.get("existing_field_name") for action in custom_field_actions)-- prompt = prompt_engine.load_prompt(- template="generate-workflow-parameters",- custom_field_actions=custom_field_actions,- has_existing_fields=has_existing_fields,- )-- response = await app.SCRIPT_GENERATION_LLM_API_HANDLER(prompt=prompt, prompt_name="generate-workflow-parameters")-- return GeneratedFieldMapping.model_validate(response)+ return # one flag is enoughdef _generate_pydantic_schema(schema_fields: Dict[str, Dict[str, str]]) -> str:diff --git a/tests/unit/test_deterministic_field_naming.py b/tests/unit/test_deterministic_field_naming.pyindex c41efe671..ac378ca00 100644--- a/tests/unit/test_deterministic_field_naming.py+++ b/tests/unit/test_deterministic_field_naming.py@@ -100,16 +100,18 @@ def test_rule_1_jinja_reference_to_declared_param() -> None:assert pick.rule == "jinja_ref"-def test_rule_1_prefers_declared_param_over_schema_key_on_tie() -> None:- """When both sets match, declared-param beats schema-key."""+def test_rule_1_falls_through_on_multi_key_goal() -> None:+ """When the goal references multiple valid keys, Rule 1 can't disambiguate+ which INPUT_TEXT action targets which key. Fall through to Rule 3 to avoid+ collapsing all actions onto the same name (CORR-1 from debate review)."""pick = pick_field_name_for_action(action=_input_action("x", intention="pick one"),goal_template="use {{ name_declared }} and {{ name_schema }}",declared_param_keys=frozenset({"name_declared"}),upstream_schema_keys=frozenset({"name_schema"}),)- assert pick.field_name == "name_declared"- assert pick.rule == "jinja_ref"+ # Multi-key → Rule 3 fallback, not Rule 1+ assert pick.rule == "intention_derived"def test_rule_1_ignores_jinja_refs_not_in_valid_keys() -> None:diff --git a/tests/unit/test_script_generation_race_condition.py b/tests/unit/test_script_generation_race_condition.pyindex 44a783f21..859debf7e 100644--- a/tests/unit/test_script_generation_race_condition.py+++ b/tests/unit/test_script_generation_race_condition.py@@ -12,10 +12,8 @@ from typing import Anyimport pytest-from skyvern.core.script_generations import generate_workflow_parameters as gwpfrom skyvern.core.script_generations.generate_workflow_parameters import (CUSTOM_FIELD_ACTIONS,- GeneratedFieldMapping,generate_workflow_parameters_schema,hydrate_input_text_actions_with_field_names,)@@ -200,16 +198,6 @@ class TestCodeGenerationWithoutFieldName:class TestFieldMappingGeneration:"""Test the field mapping generation logic."""- def test_field_mapping_structure(self) -> None:- """Test that GeneratedFieldMapping has the expected structure."""- mapping = GeneratedFieldMapping(- field_mappings={"action_index_1": "facility_name"},- schema_fields={"facility_name": {"type": "str", "description": "The facility name"}},- )-- assert mapping.field_mappings["action_index_1"] == "facility_name"- assert mapping.schema_fields["facility_name"]["type"] == "str"-def test_action_index_to_field_mapping_key_format(self) -> None:"""Test that field mapping keys use the correct format: task_id:action_id."""task_id = "task-123"@@ -243,21 +231,9 @@ async def test_generate_workflow_parameters_schema_empty_actions(monkeypatch: py@pytest.mark.asyncioasync def test_generate_workflow_parameters_schema_with_actions(monkeypatch: pytest.MonkeyPatch) -> None:"""- Integration test: Verify that when actions are present, LLM is called.-- This confirms that when script generation runs AFTER actions are saved,- it properly generates field mappings.+ Integration test: Verify that when actions are present, the deterministic+ picker generates field mappings (SKY-8965 Phase 2: no LLM call)."""-- # Mock the LLM call to return a mapping- async def mock_generate_field_names_with_llm(custom_field_actions):- return GeneratedFieldMapping(- field_mappings={"action_index_1": "facility_name"},- schema_fields={"facility_name": {"type": "str", "description": "The facility name"}},- )-- monkeypatch.setattr(gwp, "_generate_field_names_with_llm", mock_generate_field_names_with_llm)-task_id = "task-123"action_id = "action-456"actions_by_task = {@@ -268,13 +244,13 @@ async def test_generate_workflow_parameters_schema_with_actions(monkeypatch: pytschema_code, action_field_mappings = await generate_workflow_parameters_schema(actions_by_task)- # Should have generated schema with field- assert "facility_name" in schema_code+ # Should have generated schema with a field derived from the intentionassert "GeneratedWorkflowParameters" in schema_code+ assert "enter_facility_name" in schema_code# Should have mapping for our actionassert f"{task_id}:{action_id}" in action_field_mappings- assert action_field_mappings[f"{task_id}:{action_id}"] == "facility_name"+ assert action_field_mappings[f"{task_id}:{action_id}"] == "enter_facility_name"class TestRaceConditionTimingScenario:@@ -662,19 +638,6 @@ async def test_generate_workflow_parameters_schema_skips_empty_actions_and_sets_context = SkyvernContext()skyvern_context.set(context)- # Mock the LLM call - should only be called if there are valid actions- llm_called = False-- async def mock_generate_field_names_with_llm(custom_field_actions):- nonlocal llm_called- llm_called = True- return GeneratedFieldMapping(- field_mappings={"action_index_1": "facility_name"},- schema_fields={"facility_name": {"type": "str", "description": "The facility name"}},- )-- monkeypatch.setattr(gwp, "_generate_field_names_with_llm", mock_generate_field_names_with_llm)-task_id = "task-123"# Actions with empty values - simulates race condition@@ -693,9 +656,7 @@ async def test_generate_workflow_parameters_schema_skips_empty_actions_and_sets_try:schema_code, action_field_mappings = await generate_workflow_parameters_schema(actions_by_task)- # LLM should NOT be called because action was skipped- assert not llm_called-+ # Deterministic picker skips empty-value actions (same as old LLM path)# Should return empty schemaassert "pass" in schema_codeassert action_field_mappings == {}@@ -720,15 +681,6 @@ async def test_generate_workflow_parameters_schema_with_complete_actions_no_flagcontext = SkyvernContext()skyvern_context.set(context)- # Mock the LLM call- async def mock_generate_field_names_with_llm(custom_field_actions):- return GeneratedFieldMapping(- field_mappings={"action_index_1": "facility_name"},- schema_fields={"facility_name": {"type": "str", "description": "The facility name"}},- )-- monkeypatch.setattr(gwp, "_generate_field_names_with_llm", mock_generate_field_names_with_llm)-task_id = "task-123"# Actions with complete values - no race condition@@ -747,8 +699,8 @@ async def test_generate_workflow_parameters_schema_with_complete_actions_no_flagtry:schema_code, action_field_mappings = await generate_workflow_parameters_schema(actions_by_task)- # Should have generated schema- assert "facility_name" in schema_code+ # Should have generated schema with deterministic field name+ assert "enter_facility_name" in schema_code# Context flag should NOT be set - no regeneration neededassert context.script_gen_had_incomplete_actions is Falsediff --git a/tests/unit/test_workflow_schema_field_preservation.py b/tests/unit/test_workflow_schema_field_preservation.pyindex 3f3dbe49e..c1580d4db 100644--- a/tests/unit/test_workflow_schema_field_preservation.py+++ b/tests/unit/test_workflow_schema_field_preservation.py@@ -231,43 +231,18 @@ class TestGenerateWorkflowParametersSchemaWithExistingFields:# Action 3 has no existing field - needs new name}- # Mock the LLM response- mock_llm_response = {- "field_mappings": {- "action_index_1": "preserved_username",- "action_index_2": "preserved_password",- "action_index_3": "new_field_name",- },- "schema_fields": {- "preserved_username": {"type": "str", "description": "Username"},- "preserved_password": {"type": "str", "description": "Password"},- "new_field_name": {"type": "str", "description": "New field"},- },- }-- captured_prompt = {}-- async def mock_llm_handler(prompt, prompt_name):- captured_prompt["prompt"] = prompt- captured_prompt["prompt_name"] = prompt_name- return mock_llm_response-- self.stub_app.SCRIPT_GENERATION_LLM_API_HANDLER = AsyncMock(side_effect=mock_llm_handler)-+ # SKY-8965 Phase 2: no LLM mock needed — deterministic picker handles+ # field-name assignment. Existing field assignments are preserved by the+ # picker's "existing_assignment" rule (highest priority).schema_code, field_mappings = await generate_workflow_parameters_schema(actions_by_task, existing_field_assignments)- # Verify the prompt contains the existing field names- prompt = captured_prompt["prompt"]- assert "preserved_username" in prompt- assert "preserved_password" in prompt- assert "MUST PRESERVE" in prompt or "EXISTING FIELD NAME" in prompt-- # Verify the returned field mappings include preserved names+ # Verify existing field names are preserved by the deterministic pickerassert field_mappings["task_1:act_1"] == "preserved_username"assert field_mappings["task_1:act_2"] == "preserved_password"- assert field_mappings["task_2:act_3"] == "new_field_name"+ # New action gets a deterministic name from its intention ("Enter new field")+ assert field_mappings["task_2:act_3"] == "enter_new_field"@pytest.mark.asyncioasync def test_no_existing_fields_works_normally(self):@@ -279,34 +254,15 @@ class TestGenerateWorkflowParametersSchemaWithExistingFields:}existing_field_assignments: dict[int, str] = {} # No existing fields- mock_llm_response = {- "field_mappings": {- "action_index_1": "username",- },- "schema_fields": {- "username": {"type": "str", "description": "Username field"},- },- }-- captured_prompt = {}-- async def mock_llm_handler(prompt, prompt_name):- captured_prompt["prompt"] = prompt- return mock_llm_response-- self.stub_app.SCRIPT_GENERATION_LLM_API_HANDLER = AsyncMock(side_effect=mock_llm_handler)-+ # SKY-8965 Phase 2: no LLM call, no prompt to inspect. Deterministic+ # picker falls through to Rule 3 (intention-derived).schema_code, field_mappings = await generate_workflow_parameters_schema(actions_by_task, existing_field_assignments)- # Should not contain preservation instructions when no existing fields- prompt = captured_prompt["prompt"]- # The CRITICAL rule only appears when has_existing_fields is True- assert "CRITICAL" not in prompt-- # Should still return valid mappings- assert field_mappings["task_1:act_1"] == "username"+ # Should return valid mappings with deterministic name from intention+ assert "task_1:act_1" in field_mappings+ assert field_mappings["task_1:act_1"] == "enter_username"@pytest.mark.asyncioasync def test_schema_code_includes_preserved_field_names(self):@@ -427,47 +383,23 @@ async def login_block(page: SkyvernPage, context: RunContext):# Action 3 has no existing field (new block)}- # Step 2: Mock LLM that respects the preservation instructions- mock_llm_response = {- "field_mappings": {- "action_index_1": "user_email", # Preserved- "action_index_2": "user_password", # Preserved- "action_index_3": "company_name", # New field for new block- },- "schema_fields": {- "user_email": {"type": "str", "description": "User's email address"},- "user_password": {"type": "str", "description": "User's password"},- "company_name": {"type": "str", "description": "Company name"},- },- }-- captured_prompt = {}-- async def mock_llm_handler(prompt, prompt_name):- captured_prompt["prompt"] = prompt- return mock_llm_response-- self.stub_app.SCRIPT_GENERATION_LLM_API_HANDLER = AsyncMock(side_effect=mock_llm_handler)-+ # Step 2: SKY-8965 Phase 2 — deterministic picker preserves existing+ # field names and assigns new ones from intention.schema_code, field_mappings = await generate_workflow_parameters_schema(actions_by_task, existing_field_assignments)- # Verify the prompt contains preservation instructions- prompt = captured_prompt["prompt"]- assert "user_email" in prompt, "Prompt should contain existing field name 'user_email'"- assert "user_password" in prompt, "Prompt should contain existing field name 'user_password'"- assert "MUST PRESERVE" in prompt or "EXISTING FIELD NAME" in prompt-- # Verify field mappings preserve the original names+ # Verify field mappings preserve the original names via "existing_assignment" ruleassert field_mappings["task_1:act_1"] == "user_email", "Login block email field should be preserved"assert field_mappings["task_1:act_2"] == "user_password", "Login block password field should be preserved"- assert field_mappings["task_2:act_3"] == "company_name", "New block should get new field name"# Verify schema code contains preserved field namesassert "user_email" in schema_codeassert "user_password" in schema_code- assert "company_name" in schema_code++ # New block gets a deterministic name from its intention ("Enter company name")+ assert field_mappings["task_2:act_3"] == "enter_company_name"+ assert "enter_company_name" in schema_code# The cached login block code references context.parameters['user_email']# and context.parameters['user_password'], which now match the schema!