Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Revert #10280 — accidental mass deletion of copilot code (#5525)
Andrew Neilsoncf4d378a67db
diff --git a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsxindex dac552fe8..ba7abe404 100644--- a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx+++ b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx@@ -307,6 +307,14 @@ function RunWorkflowForm({);const hasLoginBlockValidationError = loginBlocksWithoutCredentials.length > 0;+ const blockingParameterTypes = new Set([+ "boolean",+ "integer",+ "float",+ "file_url",+ "json",+ ]);+const form = useForm<RunWorkflowFormType>({mode: "onTouched",reValidateMode: "onChange",@@ -325,6 +333,13 @@ function RunWorkflowForm({},});+ const formErrors = form.formState.errors;+ const hasBlockingParameterError = workflowParameters.some(+ (param) =>+ blockingParameterTypes.has(param.workflow_parameter_type) &&+ formErrors[param.key],+ );+const runWorkflowMutation = useMutation({mutationFn: async (values: RunWorkflowFormType) => {const client = await getClient(credentialGetter);@@ -497,11 +512,7 @@ function RunWorkflowForm({const handleInvalid = (errors: FieldErrors<RunWorkflowFormType>) => {const hasBlockingErrors = workflowParameters.some((param) =>- (param.workflow_parameter_type === "boolean" ||- param.workflow_parameter_type === "integer" ||- param.workflow_parameter_type === "float" ||- param.workflow_parameter_type === "file_url" ||- param.workflow_parameter_type === "json") &&+ blockingParameterTypes.has(param.workflow_parameter_type) &&errors[param.key],);@@ -560,7 +571,9 @@ function RunWorkflowForm({<Buttontype="submit"disabled={- runWorkflowMutation.isPending || hasLoginBlockValidationError+ runWorkflowMutation.isPending ||+ hasLoginBlockValidationError ||+ hasBlockingParameterError}>{runWorkflowMutation.isPending && (diff --git a/skyvern-frontend/src/routes/workflows/components/CodeEditor.tsx b/skyvern-frontend/src/routes/workflows/components/CodeEditor.tsxindex ff18661a3..9c31bcda0 100644--- a/skyvern-frontend/src/routes/workflows/components/CodeEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/components/CodeEditor.tsx@@ -61,9 +61,11 @@ function CodeEditor({}: Props) {const viewRef = useRef<EditorView | null>(null);const [internalValue, setInternalValue] = useState(value);+ const latestValueRef = useRef(value);useEffect(() => {setInternalValue(value);+ latestValueRef.current = value;}, [value]);const debouncedOnChange = useDebouncedCallback((newValue: string) => {@@ -72,6 +74,7 @@ function CodeEditor({const handleChange = (newValue: string) => {setInternalValue(newValue);+ latestValueRef.current = newValue;debouncedOnChange(newValue);};@@ -159,7 +162,8 @@ function CodeEditor({if (!viewRef.current) viewRef.current = viewUpdate.view;}}onBlur={() => {- debouncedOnChange.flush();+ debouncedOnChange.cancel();+ onChange?.(latestValueRef.current);}}/>);diff --git a/skyvern-frontend/src/routes/workflows/components/scheduleParameters.ts b/skyvern-frontend/src/routes/workflows/components/scheduleParameters.tsindex f39472501..70f8aedbb 100644--- a/skyvern-frontend/src/routes/workflows/components/scheduleParameters.ts+++ b/skyvern-frontend/src/routes/workflows/components/scheduleParameters.ts@@ -50,10 +50,21 @@ export function validateScheduleParameters(const errors: Record<string, string> = {};for (const parameter of parameters) {if (!isScheduleParameter(parameter)) continue;- if (!isRequired(parameter)) continue;const value = values[parameter.key];- if (isMissingRequiredValue(parameter, value)) {+ if (isRequired(parameter) && isMissingRequiredValue(parameter, value)) {errors[parameter.key] = "Required";+ continue;+ }+ if (+ parameter.workflow_parameter_type === "json" &&+ typeof value === "string" &&+ value.trim() !== ""+ ) {+ try {+ JSON.parse(value);+ } catch {+ errors[parameter.key] = "Invalid JSON";+ }}}return errors;diff --git a/skyvern/core/script_generations/deterministic_field_naming.py b/skyvern/core/script_generations/deterministic_field_naming.pynew file mode 100644index 000000000..197645efe--- /dev/null+++ b/skyvern/core/script_generations/deterministic_field_naming.py@@ -0,0 +1,231 @@+"""Deterministic field-name picker for workflow-parameter synthesis.++This is the replacement for `_generate_field_names_with_llm` in+`generate_workflow_parameters.py`. In Phase 1 it ships as a pure,+side-effect-free function that is NOT yet wired into the hot path — the+LLM call remains the source of truth until the validation guard in+`generate_script.py` gives us the production signal to flip the switch.++The picker implements three rules in priority order for every INPUT_TEXT,+UPLOAD_FILE, and SELECT_OPTION action observed during a workflow run. See+SKY-8965 for the motivating smoke-test repro (phantom+`preprint_search_term` on a single-block search workflow whose navigation+goal embedded the search term as a literal).++Rule precedence:+ 1. Jinja-reference rule — the unrendered `navigation_goal` template+ contains `{{ key }}` where `key` is in the valid-keys set+ (declared workflow parameters ∪ upstream block schema keys).+ Use `key` directly; no synthesis.+ 2. Upstream-schema rule — the action's value equals a literal value+ associated with an upstream block's `data_schema.properties` key.+ Use that key; no synthesis.+ 3. Intention-derived rule — deterministic snake_case sanitization of+ the action's `intention` text. Last-resort synthesis; guaranteed+ deterministic across runs with the same input.++Rules 1 and 2 both produce field names that are already in the valid-keys+set and therefore will never trip the validation guard. Rule 3 produces+synthesized names that the guard will eventually reject in Phase 2, forcing+workflow authors to declare their parameters up front or accept an+ai='proactive' fallback in the script.+"""++from __future__ import annotations++import re+from dataclasses import dataclass+from typing import Any++from skyvern.webeye.actions.actions import ActionType++CUSTOM_FIELD_ACTIONS: tuple[ActionType, ...] = (+ ActionType.INPUT_TEXT,+ ActionType.UPLOAD_FILE,+ ActionType.SELECT_OPTION,+)+++@dataclass(frozen=True)+class FieldPick:+ """Result of picking a field strategy for a single action.++ `rule` is the rule that fired: "jinja_ref" | "upstream_schema" |+ "intention_derived". The caller can use this to emit telemetry on+ rule-distribution and to branch in Phase 2 (e.g. Rule 3 becomes+ inline-literal vs ai-proactive instead of synthesis).+ """++ field_name: str+ rule: str+ description: str | None = None+++# `{{ name }}` or `{{ name.attr }}` or `{{ name | filter }}` — we only+# care about the root identifier.+_JINJA_ROOT_RE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\b")++# Sanitize an intention string down to a valid snake_case identifier.+_NON_IDENT = re.compile(r"[^a-z0-9_]+")+_REPEAT_UNDERSCORE = re.compile(r"_+")+++def sanitize_intention_to_field_name(intention: str, fallback: str = "unknown_field") -> str:+ """Deterministic sanitization of an intention string to a Python identifier.++ Mirrors the fallback pattern commented out in+ `hydrate_input_text_actions_with_field_names`, now promoted to first-class.+ """+ if not intention:+ return fallback+ lowered = intention.lower().strip()+ # Replace anything that isn't a-z0-9_ with underscores+ cleaned = _NON_IDENT.sub("_", lowered)+ # Collapse repeated underscores, strip leading/trailing+ cleaned = _REPEAT_UNDERSCORE.sub("_", cleaned).strip("_")+ if not cleaned:+ return fallback+ # Ensure it doesn't start with a digit+ if cleaned[0].isdigit():+ cleaned = f"f_{cleaned}"+ # Cap length — intentions can be paragraphs+ return cleaned[:60]+++def extract_jinja_root_names(template: str) -> set[str]:+ """Return the set of root identifiers referenced in `{{ ... }}` blocks.++ Example:+ >>> extract_jinja_root_names("Search for {{ query }} then {{ user.name }}")+ {"query", "user"}+ """+ if not template:+ return set()+ return set(_JINJA_ROOT_RE.findall(template))+++def pick_field_name_for_action(+ *,+ action: dict[str, Any],+ goal_template: str,+ declared_param_keys: frozenset[str],+ upstream_schema_keys: frozenset[str],+ existing_field_name: str | None = None,+) -> FieldPick:+ """Pick a field name for a single custom-field action.++ Args:+ action: INPUT_TEXT / UPLOAD_FILE / SELECT_OPTION action dict. Must+ carry `intention` and (depending on action_type) `text` / `file_url` /+ `option`.+ goal_template: Unrendered `navigation_goal` string for the task this+ action belongs to. Must NOT be the rendered form — otherwise the+ jinja-reference rule cannot tell a real parameter from a literal.+ declared_param_keys: Workflow-declared parameter keys+ (`workflow.workflow_definition.parameters[].key`).+ upstream_schema_keys: Keys collected from upstream blocks'+ `data_schema.properties` (output-parameter keys are collected+ separately via `_collect_declared_param_keys` at the workflow level).+ existing_field_name: Pre-existing assignment that must be preserved+ across regenerations (e.g., from a cached block's schema). When set,+ it wins unconditionally — regenerations must not rename fields that+ cached code references.++ Returns:+ FieldPick with the chosen name, the rule that fired, and (when the+ picker used jinja/schema rules) an optional description for the+ generated Pydantic field.++ Never raises. The fallback path is guaranteed to produce a valid+ identifier from the action's intention.+ """+ # Preservation wins: cached code already references this name.+ if existing_field_name:+ return FieldPick(field_name=existing_field_name, rule="existing_assignment")++ valid_keys = declared_param_keys | upstream_schema_keys+ referenced_keys = extract_jinja_root_names(goal_template)++ # Rule 1: jinja reference to a declared or schema key.+ intersection = referenced_keys & valid_keys+ if intersection:+ # When the goal references multiple keys we can't disambiguate here+ # (we don't know which INPUT_TEXT action targets which field). Prefer+ # a declared parameter over a schema key, then alphabetical for+ # determinism.+ # TODO(SKY-8965 Phase 2): before wiring this picker into the hot path,+ # disambiguate multi-key goals by falling through to Rule 3 (or a new+ # action-intention → key matcher) when `len(intersection) > 1`.+ # Otherwise every INPUT_TEXT action in a multi-field form gets the same+ # alphabetically-first name and the synthesized schema collapses fields.+ declared_matches = intersection & declared_param_keys+ preferred = sorted(declared_matches) if declared_matches else sorted(intersection)+ return FieldPick(field_name=preferred[0], rule="jinja_ref")++ # Rule 2: value matches an upstream schema key name or is clearly keyed+ # by intention to one. Conservative check: if the intention text mentions+ # a valid schema key (substring match, case-insensitive), bind to it.+ intention = (action.get("intention") or "").lower()+ for key in sorted(upstream_schema_keys):+ if key.lower() in intention:+ return FieldPick(field_name=key, rule="upstream_schema")++ # Rule 3: deterministic fallback derived from intention.+ name = sanitize_intention_to_field_name(intention)+ description = action.get("intention") or f"Value for {name}"+ return FieldPick(field_name=name, rule="intention_derived", description=description)+++def pick_field_names_for_actions(+ *,+ actions_by_task: dict[str, list[dict[str, Any]]],+ goal_template_by_task: dict[str, str],+ declared_param_keys: frozenset[str],+ upstream_schema_keys: frozenset[str],+ existing_field_assignments: dict[int, str] | None = None,+) -> dict[str, FieldPick]:+ """Bulk version: pick field names for every custom-field action.++ Returns a mapping `"{task_id}:{action_id}" → FieldPick`. Actions without+ a usable value are skipped — same behaviour as the existing LLM path in+ `generate_workflow_parameters_schema`.++ This function is pure. It does no I/O and makes no LLM calls.+ """+ existing_field_assignments = existing_field_assignments or {}+ picks: dict[str, FieldPick] = {}+ action_counter = 0++ for task_id, actions in actions_by_task.items():+ goal = goal_template_by_task.get(task_id, "")+ for action in actions:+ action_type = action.get("action_type", "")+ if action_type not in CUSTOM_FIELD_ACTIONS:+ continue++ if action_type == ActionType.INPUT_TEXT:+ value = action.get("text", "")+ elif action_type == ActionType.UPLOAD_FILE:+ value = action.get("file_url", "")+ elif action_type == ActionType.SELECT_OPTION:+ value = action.get("option", "")+ else:+ value = ""++ if not value:+ continue++ action_counter += 1+ existing = existing_field_assignments.get(action_counter)++ pick = pick_field_name_for_action(+ action=action,+ goal_template=goal,+ declared_param_keys=declared_param_keys,+ upstream_schema_keys=upstream_schema_keys,+ existing_field_name=existing,+ )+ key = f"{task_id}:{action.get('action_id', '')}"+ picks[key] = pick++ return picksdiff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.pyindex 30578a71e..28681999b 100644--- a/skyvern/core/script_generations/generate_script.py+++ b/skyvern/core/script_generations/generate_script.py@@ -7,6 +7,7 @@ Generate a runnable Skyvern workflow script.from __future__ import annotationsimport hashlib+import jsonimport keywordimport refrom collections import deque@@ -24,6 +25,11 @@ from skyvern.core.script_generations.generate_workflow_parameters import (generate_workflow_parameters_schema,hydrate_input_text_actions_with_field_names,)+from skyvern.core.script_generations.parameter_reference_guard import (+ HallucinatedParameterError,+ log_or_raise_guard_result,+ validate_context_parameter_refs,+)from skyvern.forge import appfrom skyvern.schemas.workflows import FileStorageTypefrom skyvern.utils.strings import sanitize_identifier@@ -54,6 +60,77 @@ class CodeGenResult:blocks_failed: int+# ----- SKY-8965: valid-keys collection for the parameter-reference guard --------++# `\s+` tolerates any indentation a future formatter might pick; `[^=]+?` for+# the type annotation captures complex types like `Optional[str]` and `list[str]`+# without silently dropping those fields from the valid-keys set.+_SCHEMA_FIELD_RE = re.compile(r"^\s+(\w+)\s*:\s*[^=]+?=\s*Field\(", re.MULTILINE)+++def _collect_declared_param_keys(workflow: dict[str, Any]) -> frozenset[str]:+ """Return the set of all parameter keys from the workflow definition.++ Includes workflow, output, and context parameters — any parameter type whose+ key can legally appear as `context.parameters['key']` at runtime. Secret+ parameter types (aws_secret, bitwarden_*, etc.) are included too since their+ presence in the valid-keys set only reduces false-positive guard violations.+ """+ keys: set[str] = set()+ defn = workflow.get("workflow_definition") or {}+ if not isinstance(defn, dict):+ return frozenset()+ for param in defn.get("parameters") or []:+ if not isinstance(param, dict):+ continue+ key = param.get("key")+ if key:+ keys.add(key)+ return frozenset(keys)+++def _collect_upstream_schema_keys(blocks: list[dict[str, Any]]) -> frozenset[str]:+ """Return the set of keys introduced by upstream blocks' data_schema.++ Extracts `properties` keys from any block's `data_schema` (extract-info+ blocks and task blocks with a `data_extraction_goal`). Handles both dict+ schemas and JSON-encoded string schemas (BlockYAML allows either). Recurses+ into ForLoop children (`loop_blocks`) since extract-info nested inside a+ loop still contributes keys referenced by the generated script.++ Phase 1 treats these as globally valid (a reference to `invoice_date` is OK+ anywhere in the script, not just blocks that come after the block that+ declares it). Phase 2 may tighten this to per-block ordering.+ """+ keys: set[str] = set()+ for block in blocks or []:+ schema = block.get("data_schema")+ if isinstance(schema, str):+ try:+ schema = json.loads(schema)+ except (ValueError, TypeError):+ schema = None+ if isinstance(schema, dict):+ props = schema.get("properties")+ if isinstance(props, dict):+ keys.update(str(k) for k in props.keys())+ nested = block.get("loop_blocks")+ if isinstance(nested, list):+ keys |= _collect_upstream_schema_keys(nested)+ return frozenset(keys)+++def _collect_synthesized_field_keys(schema_code: str) -> frozenset[str]:+ """Extract field names from a generated `GeneratedWorkflowParameters` class.++ Parses the schema code by regex for the `name: type = Field(...)` pattern.+ Returns an empty frozenset for the empty-schema case (`pass` body).+ """+ if not schema_code:+ return frozenset()+ return frozenset(_SCHEMA_FIELD_RE.findall(schema_code))++def _build_existing_field_assignments(blocks: list[dict[str, Any]],actions_by_task: dict[str, list[dict[str, Any]]],@@ -3366,7 +3443,36 @@ async def generate_workflow_script_python_code(]module = cst.Module(body=module_body)- return CodeGenResult(source_code=module.code, blocks_created=blocks_created, blocks_failed=blocks_failed)+ source_code = module.code++ # SKY-8965 Phase 1: validate `context.parameters[X]` references against the+ # expanded valid-keys set (declared params ∪ upstream block schema keys ∪+ # synthesized `GeneratedWorkflowParameters` fields). Phase 1 logs violations+ # only — we want production telemetry before raising in Phase 2. Wrapped in+ # try/except so a guard bug can never break script generation.+ try:+ declared_keys = _collect_declared_param_keys(workflow)+ upstream_keys = _collect_upstream_schema_keys(blocks)+ synthesized_keys = _collect_synthesized_field_keys(generated_schema)+ guard_result = validate_context_parameter_refs(+ code=source_code,+ declared_param_keys=declared_keys,+ upstream_schema_keys=upstream_keys,+ synthesized_keys=synthesized_keys,+ )+ log_or_raise_guard_result(+ guard_result,+ raise_on_violation=False,+ workflow_permanent_id=workflow.get("workflow_permanent_id"),+ workflow_run_id=run_id,+ )+ except HallucinatedParameterError:+ # Phase 2 will raise this intentionally — don't swallow it.+ raise+ except Exception:+ LOG.warning("parameter_reference_guard_failed_to_run", exc_info=True)++ return CodeGenResult(source_code=source_code, blocks_created=blocks_created, blocks_failed=blocks_failed)async def create_or_update_script_block(diff --git a/skyvern/core/script_generations/parameter_reference_guard.py b/skyvern/core/script_generations/parameter_reference_guard.pynew file mode 100644index 000000000..7f4faba49--- /dev/null+++ b/skyvern/core/script_generations/parameter_reference_guard.py@@ -0,0 +1,167 @@+"""Validation guard for undeclared `context.parameters[X]` references.++Catches the phantom-parameter pattern at the end of initial script generation+(SKY-8965). Complements `ScriptReviewer._validate_parameter_references`, which+runs only during the reviewer retry path — this guard runs at the initial-gen+pipeline stage, where the existing reviewer validator never sees the code.++Phase 1 (this PR): guard runs in WARNING mode. It logs violations with full+context but does not raise. This gives us a production signal in Datadog for+hallucination frequency before we decide to escalate.++Phase 2 (follow-up): flip the guard to raise on violation. At that point+workflow authors either declare their parameters up front, or the script+generator falls back to `ai='proactive'` for the undeclared fields (Phase 2+change in `generate_workflow_parameters.py`).+"""++from __future__ import annotations++import re+from dataclasses import dataclass, field++import structlog++LOG = structlog.get_logger(__name__)+++# Matches both subscript (`context.parameters['key']`) and dict-get+# (`context.parameters.get('key')` or `.get('key', default)`) access. The+# current initial-gen emits subscripts only, but the reviewer retry path and+# its prompt templates use `.get()`, so the guard handles both to stay+# consistent with any future unification with ScriptReviewer._PARAM_REF_RE.+# Attribute access (`context.parameters.key`) is NOT emitted anywhere today,+# so it's intentionally excluded — add it if Phase 2 introduces it.+_PARAM_REF_RE = re.compile(r"""context\.parameters(?:\[['"](\w+)['"]\]|\.get\(\s*['"](\w+)['"]\s*(?:,[^)]*)?\))""")+++@dataclass(frozen=True)+class UndeclaredReference:+ key: str+ # One example location for debugging; we don't collect every occurrence.+ example_line: str+++@dataclass+class GuardResult:+ """Result of validating a generated script against the valid-keys set."""++ valid: bool+ undeclared_refs: list[UndeclaredReference] = field(default_factory=list)+ referenced_keys: frozenset[str] = frozenset()+ valid_keys: frozenset[str] = frozenset()++ def format_error(self) -> str:+ """Human-readable error string for logs / raised exceptions."""+ invalid_list = ", ".join(repr(r.key) for r in self.undeclared_refs)+ valid_list = ", ".join(repr(k) for k in sorted(self.valid_keys)) or "(none)"+ return (+ f"Generated script references undeclared workflow parameters: {invalid_list}. "+ f"Valid keys are: {valid_list}. This usually means the synthesis LLM invented a "+ f"field name for a value that was a literal in the navigation goal. See SKY-8965."+ )+++def _collect_refs(code: str) -> list[UndeclaredReference]:+ """Walk the code line-by-line and collect every context.parameters[...] reference.++ Comments are skipped. Multiple references per line produce multiple entries.+ """+ refs: list[UndeclaredReference] = []+ for line in code.split("\n"):+ stripped = line.lstrip()+ if stripped.startswith("#"):+ continue+ for match in _PARAM_REF_RE.finditer(line):+ # Group 1 = subscript form, group 2 = .get() form; exactly one fires.+ key = match.group(1) or match.group(2)+ if key:+ refs.append(UndeclaredReference(key=key, example_line=line.strip()))+ return refs+++def validate_context_parameter_refs(+ *,+ code: str,+ declared_param_keys: frozenset[str],+ upstream_schema_keys: frozenset[str],+ synthesized_keys: frozenset[str] = frozenset(),+) -> GuardResult:+ """Validate that every `context.parameters[X]` in `code` references a known key.++ The valid-keys set is the union of:+ - `declared_param_keys`: every key declared in+ `workflow_definition.parameters` regardless of parameter_type (workflow,+ output, context, secret — all are legal `context.parameters[X]` targets+ at runtime).+ - `upstream_schema_keys`: keys exposed by upstream extract-info / task+ blocks via their `data_schema.properties` (recursed into ForLoop+ children).+ - `synthesized_keys`: keys present in the generated `GeneratedWorkflowParameters`+ Pydantic class. Callable from Phase 1 to stay compatible with the current+ LLM synthesis path; Phase 2 will shrink this to empty and rely entirely+ on the first two sets.++ Args:+ code: The Python source of the generated script (usually `main.py` ++ concatenated block `.skyvern` files). Pass as a single string.+ declared_param_keys: Frozenset of declared workflow parameter keys.+ upstream_schema_keys: Frozenset of keys exposed by upstream blocks.+ synthesized_keys: Frozenset of field names currently present in the+ synthesized `GeneratedWorkflowParameters` class.++ Returns:+ A `GuardResult` with `valid=False` when any reference falls outside the+ valid set. The caller decides whether to log (Phase 1) or raise+ (Phase 2).+ """+ refs = _collect_refs(code)+ referenced_keys = frozenset(r.key for r in refs)+ valid_keys = declared_param_keys | upstream_schema_keys | synthesized_keys++ undeclared = [r for r in refs if r.key not in valid_keys]+ return GuardResult(+ valid=not undeclared,+ undeclared_refs=undeclared,+ referenced_keys=referenced_keys,+ valid_keys=valid_keys,+ )+++class HallucinatedParameterError(RuntimeError):+ """Raised in Phase 2 when the guard rejects the generated script.++ Phase 1 does not raise; it only logs. The exception class is defined now+ so downstream code can be written against it ahead of the flip.+ """++ def __init__(self, result: GuardResult) -> None:+ super().__init__(result.format_error())+ self.result = result+++def log_or_raise_guard_result(+ result: GuardResult,+ *,+ raise_on_violation: bool,+ workflow_permanent_id: str | None = None,+ workflow_run_id: str | None = None,+) -> None:+ """Apply the Phase 1 / Phase 2 policy to a guard result.++ Phase 1: `raise_on_violation=False`, we only log a warning.+ Phase 2: `raise_on_violation=True`, we also raise `HallucinatedParameterError`.+ """+ if result.valid:+ return++ LOG.warning(+ "parameter_reference_guard_violation",+ workflow_permanent_id=workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ undeclared_refs=sorted(r.key for r in result.undeclared_refs),+ valid_keys=sorted(result.valid_keys),+ sky_ticket="SKY-8965",+ )+ if raise_on_violation:+ raise HallucinatedParameterError(result)diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex b1b6ed22c..c6904b8f0 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -5113,7 +5113,6 @@ class ForgeAgent:summary_kwargs: dict[str, Any] = {"data_extraction_goal": task.data_extraction_goal,"data_extraction_schema": capped_schema,- "current_url": scraped_page.url,"local_datetime": local_datetime_str,}prompt = prompt_engine.load_prompt("data-extraction-summary", **summary_kwargs)@@ -5124,25 +5123,27 @@ class ForgeAgent:kwargs=summary_kwargs,)- # Cache the summary LLM call — the inputs (goal, schema, URL) are- # identical across download-loop iterations that revisit the same page.- # The `try` is narrowed to just compute_cache_key + lookup so a- # downstream log failure can't double-count as a lookup_error.+ # Cache the summary LLM call — the inputs (goal, schema) are identical+ # across loop iterations that share an extraction configuration, even+ # when each iteration navigates to a different URL. The `try` is+ # narrowed to just compute_cache_key + lookup so a downstream log+ # failure can't double-count as a lookup_error.workflow_run_id = context.workflow_run_id if context else Nonecache_key: str | None = Nonelookup_result: extraction_cache.LookupResult | None = Nonetry:- # data-extraction-summary is goal-agnostic: only goal/schema/URL/datetime- # affect the rendered prompt, so everything else is intentionally omitted.- # Hash the post-ceiling value for `data_extraction_schema` — when the- # rendered prompt exceeds PROMPT_HARD_CEILING_TOKENS the ceiling drops- # that field to None, so two oversized-schema requests that both get- # dropped produce an identical LLM prompt and must share a cache key.+ # data-extraction-summary plans *what* will be extracted from+ # goal + schema; only those inputs, plus the date (via datetime),+ # affect the rendered prompt, so everything else is intentionally+ # omitted. Hash the post-ceiling value for `data_extraction_schema`+ # — when the rendered prompt exceeds PROMPT_HARD_CEILING_TOKENS the+ # ceiling drops that field to None, so two oversized-schema requests+ # that both get dropped produce an identical LLM prompt and must+ # share a cache key.cache_key = extraction_cache.compute_cache_key(call_path="agent",data_extraction_goal=task.data_extraction_goal,extracted_information_schema=post_ceiling_kwargs["data_extraction_schema"],- current_url=scraped_page.url,local_datetime=local_datetime_str,llm_key=None,)diff --git a/skyvern/forge/prompts/skyvern/data-extraction-summary.j2 b/skyvern/forge/prompts/skyvern/data-extraction-summary.j2index 55fc77921..a2773bf4d 100644--- a/skyvern/forge/prompts/skyvern/data-extraction-summary.j2+++ b/skyvern/forge/prompts/skyvern/data-extraction-summary.j2@@ -5,8 +5,6 @@ Reply in JSON format with the following keys:"summary": str, // Summary of the data you will extract within one sentence. Be precise and concise.}-The URL of the page you're on right now is `{{ current_url }}`.-Data extraction goal:```{{ data_extraction_goal }}diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.pynew file mode 100644index 000000000..be0fdc20f--- /dev/null+++ b/tests/unit/test_copilot_session_injection.py@@ -0,0 +1,488 @@+"""Tests for copilot session injection and output contract adapters."""++from __future__ import annotations++import asyncio+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest++from skyvern.cli.core import session_manager+from skyvern.cli.core.result import BrowserContext as MCPBrowserContext+from skyvern.cli.core.session_manager import SessionState, scoped_session+from skyvern.forge.sdk.copilot.runtime import AgentContext, mcp_to_copilot++[email protected](autouse=True)+def _reset_session_state() -> None:+ session_manager._current_session.set(None)+ session_manager._global_session = None+ session_manager.set_stateless_http_mode(False)+++def _make_stream() -> MagicMock:+ stream = MagicMock()+ stream.is_disconnected = AsyncMock(return_value=False)+ return stream+++def _make_ctx(**overrides: Any) -> AgentContext:+ defaults = dict(+ organization_id="org-1",+ workflow_id="wf-1",+ workflow_permanent_id="wfp-1",+ workflow_yaml="",+ browser_session_id="pbs_test_123",+ stream=_make_stream(),+ api_key="sk-test-key",+ )+ defaults.update(overrides)+ return AgentContext(**defaults)+++async def test_scoped_session_pushes_and_restores() -> None:+ """scoped_session sets the ContextVar within scope and restores on exit."""+ # ContextVar starts as None (from fixture reset)+ assert session_manager._current_session.get() is None++ injected = SessionState(+ browser=MagicMock(),+ context=MCPBrowserContext(mode="cloud_session", session_id="pbs_injected"),+ )++ async with scoped_session(injected):+ inside = session_manager.get_current_session()+ assert inside is injected+ assert inside.context.session_id == "pbs_injected"++ after = session_manager._current_session.get()+ # Should be restored to None (the value before scoped_session set it)+ assert after is None+++async def test_scoped_session_does_not_touch_global() -> None:+ """scoped_session must NOT mutate _global_session."""+ session_manager._global_session = None++ injected = SessionState(+ browser=MagicMock(),+ context=MCPBrowserContext(mode="cloud_session", session_id="pbs_test"),+ )++ async with scoped_session(injected):+ pass++ assert session_manager._global_session is None+++async def test_scoped_session_concurrent_isolation() -> None:+ """Two concurrent scoped_session calls don't interfere with each other."""+ results: dict[str, str | None] = {}+ both_entered = asyncio.Event()+ entered_count = 0++ async def worker(session_id: str) -> None:+ nonlocal entered_count+ state = SessionState(+ browser=MagicMock(),+ context=MCPBrowserContext(mode="cloud_session", session_id=session_id),+ )+ async with scoped_session(state):+ entered_count += 1+ if entered_count == 2:+ both_entered.set()+ # Wait until BOTH workers are inside their scope before reading —+ # this guarantees ContextVar isolation is the only thing separating them.+ await both_entered.wait()+ current = session_manager.get_current_session()+ results[session_id] = current.context.session_id if current.context else None++ await asyncio.gather(worker("pbs_a"), worker("pbs_b"))++ assert results["pbs_a"] == "pbs_a"+ assert results["pbs_b"] == "pbs_b"+ assert session_manager._global_session is None+++def test_mcp_to_copilot_success() -> None:+ mcp_result = {+ "ok": True,+ "action": "skyvern_navigate",+ "browser_context": {"mode": "cloud_session", "session_id": "pbs_1"},+ "data": {"url": "https://example.com", "title": "Example"},+ "timing_ms": {"total": 500},+ "artifacts": [],+ }+ result = mcp_to_copilot(mcp_result)+ assert result["ok"] is True+ assert result["data"]["url"] == "https://example.com"+ assert "action" not in result+ assert "browser_context" not in result+ assert "timing_ms" not in result+ assert "artifacts" not in result+++def test_mcp_to_copilot_error() -> None:+ mcp_result = {+ "ok": False,+ "error": {"code": "NO_ACTIVE_BROWSER", "message": "No browser", "hint": "Create one"},+ }+ result = mcp_to_copilot(mcp_result)+ assert result["ok"] is False+ assert "No browser" in result["error"]+ assert "Create one" in result["error"]+++class TestMcpBrowserContextBridge:+ """Bridge-specific behavior of mcp_browser_context (not scoped_session).++ Covers: copilot session registry, API-key override install/reset, and the+ teardown guarantees that must hold under every failure mode.+ """++ def _install_happy_path_mocks(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> tuple[MagicMock, MagicMock, MagicMock, MagicMock, list[Any]]:+ import skyvern.forge.sdk.copilot.runtime as runtime++ browser_state = MagicMock()+ browser_state.browser_context = MagicMock()+ manager = MagicMock()+ manager.get_browser_state = AsyncMock(return_value=browser_state)+ monkeypatch.setattr(runtime.app, "PERSISTENT_SESSIONS_MANAGER", manager)++ monkeypatch.setattr(runtime, "get_skyvern", lambda: MagicMock())+ monkeypatch.setattr(runtime, "SkyvernBrowser", lambda *a, **kw: MagicMock())+ monkeypatch.setattr(runtime, "get_active_api_key", lambda: "sk-test-key")+ monkeypatch.setattr(runtime, "hash_api_key_for_cache", lambda k: "hash_" + k)++ override_token = object()+ override_calls: list[Any] = []+ monkeypatch.setattr(+ runtime, "set_api_key_override", lambda k: (override_calls.append(("set", k)), override_token)[1]+ )+ monkeypatch.setattr(runtime, "reset_api_key_override", lambda t: override_calls.append(("reset", t)))++ register_mock = MagicMock()+ unregister_mock = MagicMock()+ monkeypatch.setattr(runtime, "register_copilot_session", register_mock)+ monkeypatch.setattr(runtime, "unregister_copilot_session", unregister_mock)++ return manager, register_mock, unregister_mock, browser_state, override_calls++ @pytest.mark.asyncio+ async def test_happy_path_registers_session_and_balances_unregister(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context++ _, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)++ ctx = _make_ctx()+ async with mcp_browser_context(ctx):+ # Inside the context, the bridge has registered a SessionState whose+ # session_id matches the agent context — this is the public contract+ # that resolve_browser(session_id=...) relies on.+ args = register_mock.call_args.args+ assert args[0] == ctx.browser_session_id+ registered_state = args[1]+ assert isinstance(registered_state, SessionState)+ assert registered_state.context.session_id == ctx.browser_session_id++ assert register_mock.call_count == 1+ assert unregister_mock.call_count == 1+ unregister_mock.assert_called_with(ctx.browser_session_id)+ # Override installed then reset.+ assert [c[0] for c in override_calls] == ["set", "reset"]++ @pytest.mark.asyncio+ async def test_missing_browser_context_raises_without_leaking_session_id(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ import skyvern.forge.sdk.copilot.runtime as runtime+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context++ manager = MagicMock()+ manager.get_browser_state = AsyncMock(return_value=None)+ monkeypatch.setattr(runtime.app, "PERSISTENT_SESSIONS_MANAGER", manager)++ ctx = _make_ctx()+ with pytest.raises(RuntimeError, match="No browser context for copilot session") as exc_info:+ async with mcp_browser_context(ctx):+ pytest.fail("should not enter body")++ # Session id must not leak into the user/LLM-visible exception message.+ assert ctx.browser_session_id not in str(exc_info.value)++ @pytest.mark.asyncio+ async def test_exception_during_yield_still_tears_down(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context++ _, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)++ class Boom(RuntimeError):+ pass++ ctx = _make_ctx()+ with pytest.raises(Boom):+ async with mcp_browser_context(ctx):+ raise Boom("caller raised inside context")++ # Both teardown paths must fire even when the caller raises.+ assert register_mock.call_count == 1+ assert unregister_mock.call_count == 1+ assert [c[0] for c in override_calls] == ["set", "reset"]++ @pytest.mark.asyncio+ async def test_setup_phase_failure_still_resets_api_key_override(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """If get_skyvern raises AFTER set_api_key_override, the override must+ still be reset so the request-scoped API key does not leak across+ requests."""+ import skyvern.forge.sdk.copilot.runtime as runtime+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context++ _, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)++ def _raising_get_skyvern() -> Any:+ raise RuntimeError("skyvern client unavailable")++ monkeypatch.setattr(runtime, "get_skyvern", _raising_get_skyvern)++ ctx = _make_ctx()+ with pytest.raises(RuntimeError, match="skyvern client unavailable"):+ async with mcp_browser_context(ctx):+ pytest.fail("should not enter body")++ # Registration never happened because setup failed before register_copilot_session.+ assert register_mock.call_count == 0+ assert unregister_mock.call_count == 0+ # But the override must have been set AND reset.+ assert [c[0] for c in override_calls] == ["set", "reset"]+++class TestScreenshotAdapter:+ @pytest.mark.asyncio+ async def test_screenshot_post_hook_reshapes_data_with_url_and_title(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _screenshot_post_hook++ ctx = _make_ctx()+ raw = {"browser_context": {"url": "https://example.com", "title": "Example"}}+ result = {+ "ok": True,+ "data": {"data": "iVBOR...", "mime": "image/png", "bytes": 1234},+ }++ adapted = await _screenshot_post_hook(result, raw, ctx)++ assert adapted["data"]["screenshot_base64"] == "iVBOR..."+ assert adapted["data"]["url"] == "https://example.com"+ assert adapted["data"]["title"] == "Example"+++class TestNavigateAdapter:+ @pytest.mark.asyncio+ async def test_navigate_post_hook_lifts_url_and_adds_next_step(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _navigate_post_hook++ ctx = _make_ctx()+ raw = {"browser_context": {"url": "https://example.com", "title": "Example"}}+ result = {"ok": True, "data": {"url": "https://example.com", "title": "Example"}}++ adapted = await _navigate_post_hook(result, raw, ctx)++ assert adapted["ok"] is True+ assert adapted["url"] == "https://example.com"+ assert "next_step" in adapted+ assert "data" not in adapted+++class TestClickAdapter:+ @pytest.mark.asyncio+ async def test_click_post_hook_reshapes_data_with_url_and_title(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _click_post_hook++ ctx = _make_ctx()+ raw = {"browser_context": {"url": "https://ex.com", "title": "Page"}}+ result = {+ "ok": True,+ "data": {"selector": "#btn", "intent": None, "sdk_equivalent": "..."},+ }++ adapted = await _click_post_hook(result, raw, ctx)++ assert adapted["data"]["selector"] == "#btn"+ assert adapted["data"]["url"] == "https://ex.com"+ assert adapted["data"]["title"] == "Page"+ assert "sdk_equivalent" not in adapted["data"]+++class TestTypeTextAdapter:+ @pytest.mark.asyncio+ async def test_type_text_post_hook_renames_text_length_to_typed_length(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _type_text_post_hook++ ctx = _make_ctx()+ raw = {"browser_context": {"url": "https://ex.com"}}+ result = {+ "ok": True,+ "data": {"selector": "#email", "text_length": 15, "sdk_equivalent": "..."},+ }++ adapted = await _type_text_post_hook(result, raw, ctx)++ assert adapted["data"]["selector"] == "#email"+ assert adapted["data"]["typed_length"] == 15+ assert adapted["data"]["url"] == "https://ex.com"+ assert "text_length" not in adapted["data"]+++class TestEvaluateAdapter:+ def test_evaluate_copilot_contract(self) -> None:+ mcp_result = {+ "ok": True,+ "action": "skyvern_evaluate",+ "data": {+ "result": {"title": "Test"},+ "sdk_equivalent": "await page.evaluate(...)",+ },+ "browser_context": {"mode": "cloud_session"},+ }+ result = mcp_to_copilot(mcp_result)+ assert result["data"]["result"] == {"title": "Test"}+++class TestUpdateWorkflowDirect:+ @pytest.mark.asyncio+ async def test_calls_internal_with_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """update_workflow uses direct path even when api_key is set."""+ from skyvern.forge.sdk.copilot.tools import _update_workflow++ ctx = _make_ctx(api_key="sk-test-key", workflow_permanent_id="wpid_abc123")++ mock_workflow = MagicMock()+ mock_workflow.title = "Test"+ mock_workflow.description = ""+ mock_workflow.workflow_definition = MagicMock()+ mock_workflow.workflow_definition.blocks = [MagicMock(), MagicMock()]++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",+ lambda **kwargs: mock_workflow,+ )++ mock_wf_service = MagicMock()+ mock_wf_service.update_workflow_definition = AsyncMock()+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.app.WORKFLOW_SERVICE", mock_wf_service)++ yaml_str = "title: Test\nworkflow_definition:\n blocks: []"+ result = await _update_workflow({"workflow_yaml": yaml_str}, ctx)++ assert result["ok"] is True+ assert result["data"]["block_count"] == 2+ assert result["_workflow"] is mock_workflow+ assert ctx.workflow_yaml == yaml_str+ mock_wf_service.update_workflow_definition.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_calls_internal_without_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """update_workflow uses direct path when api_key is None."""+ from skyvern.forge.sdk.copilot.tools import _update_workflow++ ctx = _make_ctx(api_key=None)++ mock_workflow = MagicMock()+ mock_workflow.title = "Test"+ mock_workflow.description = ""+ mock_workflow.workflow_definition = MagicMock()+ mock_workflow.workflow_definition.blocks = []++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",+ lambda **kwargs: mock_workflow,+ )++ mock_wf_service = MagicMock()+ mock_wf_service.update_workflow_definition = AsyncMock()+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.app.WORKFLOW_SERVICE", mock_wf_service)++ result = await _update_workflow({"workflow_yaml": "title: Test"}, ctx)++ assert result["ok"] is True+ mock_wf_service.update_workflow_definition.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_yaml_parse_error_returns_failure(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """YAML parse errors return ok: false."""+ import yaml as _yaml++ from skyvern.forge.sdk.copilot.tools import _update_workflow++ ctx = _make_ctx(api_key="sk-test-key")++ def raise_yaml_error(**kwargs: Any) -> None:+ raise _yaml.YAMLError("bad yaml")++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",+ raise_yaml_error,+ )++ result = await _update_workflow({"workflow_yaml": "bad: yaml: {"}, ctx)++ assert result["ok"] is False+ assert "Workflow validation failed" in result["error"]++ @pytest.mark.asyncio+ async def test_ctx_workflow_yaml_not_updated_on_failure(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """ctx.workflow_yaml should NOT be updated when processing fails."""+ from pydantic import ValidationError as _ValidationError++ from skyvern.forge.sdk.copilot.tools import _update_workflow++ ctx = _make_ctx(+ api_key="sk-test-key",+ workflow_yaml="original yaml",+ )++ def raise_validation_error(**kwargs: Any) -> None:+ raise _ValidationError.from_exception_data(+ title="WorkflowCreateYAMLRequest",+ line_errors=[],+ )++ monkeypatch.setattr(+ "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",+ raise_validation_error,+ )++ await _update_workflow({"workflow_yaml": "new broken yaml"}, ctx)++ assert ctx.workflow_yaml == "original yaml"+++class TestWorkflowUpdatePersistence:+ def test_record_marks_persisted_even_when_workflow_has_zero_blocks(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _record_workflow_update_result++ ctx = MagicMock()+ ctx.workflow_yaml = "title: Empty workflow"+ ctx.workflow_persisted = False++ workflow = MagicMock()+ workflow.workflow_definition = MagicMock()+ workflow.workflow_definition.blocks = []++ _record_workflow_update_result(+ ctx,+ {+ "ok": True,+ "_workflow": workflow,+ "data": {"block_count": 0},+ },+ )++ assert ctx.last_workflow is workflow+ assert ctx.last_workflow_yaml == "title: Empty workflow"+ assert ctx.workflow_persisted is Truediff --git a/tests/unit/test_deterministic_field_naming.py b/tests/unit/test_deterministic_field_naming.pynew file mode 100644index 000000000..c41efe671--- /dev/null+++ b/tests/unit/test_deterministic_field_naming.py@@ -0,0 +1,246 @@+"""Tests for the deterministic field-name picker (SKY-8965 Phase 1).++See `skyvern/core/script_generations/deterministic_field_naming.py` for the+three rules under test:+ 1. jinja_ref — `{{ key }}` in unrendered goal references a known key+ 2. upstream_schema — intention text references an upstream schema key+ 3. intention_derived — deterministic fallback from the action's intention+"""++from __future__ import annotations++from skyvern.core.script_generations.deterministic_field_naming import (+ extract_jinja_root_names,+ pick_field_name_for_action,+ pick_field_names_for_actions,+ sanitize_intention_to_field_name,+)+from skyvern.webeye.actions.actions import ActionType++# --- sanitize_intention_to_field_name --------------------------------------+++def test_sanitize_preserves_valid_identifiers() -> None:+ assert sanitize_intention_to_field_name("worker_name") == "worker_name"+++def test_sanitize_lowercases_and_replaces_spaces() -> None:+ assert sanitize_intention_to_field_name("Worker Name") == "worker_name"+++def test_sanitize_strips_punctuation() -> None:+ assert sanitize_intention_to_field_name("What's your email?") == "what_s_your_email"+++def test_sanitize_collapses_repeats() -> None:+ assert sanitize_intention_to_field_name("foo bar baz") == "foo_bar_baz"+++def test_sanitize_handles_leading_digit() -> None:+ result = sanitize_intention_to_field_name("123 dollars")+ assert result.startswith("f_") and "123" in result+++def test_sanitize_caps_length() -> None:+ long = "a " * 100+ assert len(sanitize_intention_to_field_name(long)) <= 60+++def test_sanitize_empty_returns_fallback() -> None:+ assert sanitize_intention_to_field_name("") == "unknown_field"+ assert sanitize_intention_to_field_name("!!!") == "unknown_field"+++# --- extract_jinja_root_names ----------------------------------------------+++def test_jinja_root_names_simple() -> None:+ assert extract_jinja_root_names("Search for {{ query }}") == {"query"}+++def test_jinja_root_names_multiple() -> None:+ names = extract_jinja_root_names("{{ a }} and {{ b }} with {{ a }}")+ assert names == {"a", "b"}+++def test_jinja_root_names_attribute_access() -> None:+ # Only root identifier, not the attribute+ assert extract_jinja_root_names("hi {{ user.name }}") == {"user"}+++def test_jinja_root_names_with_filters() -> None:+ assert extract_jinja_root_names("{{ value | upper }}") == {"value"}+++def test_jinja_root_names_no_interpolation() -> None:+ assert extract_jinja_root_names("no jinja here") == set()+++# --- pick_field_name_for_action --------------------------------------------+++def _input_action(text: str, intention: str = "", action_id: str = "a1") -> dict:+ return {+ "action_type": ActionType.INPUT_TEXT,+ "text": text,+ "intention": intention,+ "action_id": action_id,+ }+++def test_rule_1_jinja_reference_to_declared_param() -> None:+ """If the goal contains `{{ key }}` and `key` is a declared param, use it."""+ pick = pick_field_name_for_action(+ action=_input_action("quantum computing", intention="search term"),+ goal_template="Search for preprints about {{ search_term }}.",+ declared_param_keys=frozenset({"search_term"}),+ upstream_schema_keys=frozenset(),+ )+ assert pick.field_name == "search_term"+ assert pick.rule == "jinja_ref"+++def test_rule_1_prefers_declared_param_over_schema_key_on_tie() -> None:+ """When both sets match, declared-param beats schema-key."""+ pick = pick_field_name_for_action(+ action=_input_action("x", intention="pick one"),+ goal_template="use {{ name_declared }} and {{ name_schema }}",+ declared_param_keys=frozenset({"name_declared"}),+ upstream_schema_keys=frozenset({"name_schema"}),+ )+ assert pick.field_name == "name_declared"+ assert pick.rule == "jinja_ref"+++def test_rule_1_ignores_jinja_refs_not_in_valid_keys() -> None:+ """A `{{ made_up }}` in the goal where `made_up` isn't declared → fall through."""+ pick = pick_field_name_for_action(+ action=_input_action("quantum computing", intention="search term for preprints"),+ goal_template="Search for {{ made_up_name }}",+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ )+ # No match → Rule 3 fallback+ assert pick.rule == "intention_derived"+++def test_rule_2_upstream_schema_via_intention_substring() -> None:+ """Intention text mentions an upstream schema key → use it."""+ pick = pick_field_name_for_action(+ action=_input_action("2026-04-15", intention="fill the invoice_date field"),+ goal_template="Download the invoice for the extracted date",+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset({"invoice_date"}),+ )+ assert pick.field_name == "invoice_date"+ assert pick.rule == "upstream_schema"+++def test_rule_3_fallback_from_intention() -> None:+ """No declared param, no schema match → sanitize intention."""+ pick = pick_field_name_for_action(+ action=_input_action("quantum computing", intention="Enter the search term for preprints"),+ goal_template="Search for 'quantum computing' preprints.",+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ )+ assert pick.rule == "intention_derived"+ assert pick.field_name.startswith("enter_the_search_term")+++def test_existing_field_name_wins_unconditionally() -> None:+ """A preserved assignment from a cached block beats every rule."""+ pick = pick_field_name_for_action(+ action=_input_action("X", intention="whatever"),+ goal_template="{{ completely_different_param }}",+ declared_param_keys=frozenset({"completely_different_param"}),+ upstream_schema_keys=frozenset(),+ existing_field_name="legacy_name",+ )+ assert pick.field_name == "legacy_name"+ assert pick.rule == "existing_assignment"+++# --- pick_field_names_for_actions (bulk) -----------------------------------+++def test_bulk_picks_skip_empty_values() -> None:+ actions = {+ "t1": [+ {+ "action_type": ActionType.INPUT_TEXT,+ "text": "",+ "intention": "empty",+ "action_id": "a1",+ },+ {+ "action_type": ActionType.INPUT_TEXT,+ "text": "real",+ "intention": "real value",+ "action_id": "a2",+ },+ ]+ }+ picks = pick_field_names_for_actions(+ actions_by_task=actions,+ goal_template_by_task={"t1": ""},+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ )+ assert "t1:a1" not in picks+ assert "t1:a2" in picks+++def test_bulk_skips_non_custom_field_actions() -> None:+ actions = {+ "t1": [+ {+ "action_type": ActionType.CLICK,+ "action_id": "a1",+ },+ {+ "action_type": ActionType.INPUT_TEXT,+ "text": "foo",+ "intention": "type foo",+ "action_id": "a2",+ },+ ]+ }+ picks = pick_field_names_for_actions(+ actions_by_task=actions,+ goal_template_by_task={"t1": ""},+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ )+ assert list(picks.keys()) == ["t1:a2"]+++def test_bulk_applies_existing_assignments_by_counter_order() -> None:+ actions = {+ "t1": [+ {+ "action_type": ActionType.INPUT_TEXT,+ "text": "x",+ "intention": "first",+ "action_id": "a1",+ },+ {+ "action_type": ActionType.INPUT_TEXT,+ "text": "y",+ "intention": "second",+ "action_id": "a2",+ },+ ]+ }+ # 1-indexed counter — assignment for the second action+ picks = pick_field_names_for_actions(+ actions_by_task=actions,+ goal_template_by_task={"t1": ""},+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ existing_field_assignments={2: "locked_name"},+ )+ assert picks["t1:a2"].field_name == "locked_name"+ assert picks["t1:a2"].rule == "existing_assignment"+ # First action unchanged+ assert picks["t1:a1"].rule == "intention_derived"diff --git a/tests/unit/test_parameter_reference_guard.py b/tests/unit/test_parameter_reference_guard.pynew file mode 100644index 000000000..605cb383a--- /dev/null+++ b/tests/unit/test_parameter_reference_guard.py@@ -0,0 +1,245 @@+"""Tests for the context.parameters reference guard (SKY-8965 Phase 1)."""++from __future__ import annotations++import pytest++from skyvern.core.script_generations.generate_script import (+ _collect_declared_param_keys,+ _collect_upstream_schema_keys,+)+from skyvern.core.script_generations.parameter_reference_guard import (+ HallucinatedParameterError,+ log_or_raise_guard_result,+ validate_context_parameter_refs,+)++SCRIPT_WITH_TWO_REFS = """+async def block_fn(page, context):+ await page.fill(value=context.parameters['search_term'])+ await page.fill(value=context.parameters['other_key'])+"""++SCRIPT_WITH_PHANTOM = """+async def block_fn(page, context):+ await page.fill(value=context.parameters['preprint_search_term'])+"""++SCRIPT_WITH_COMMENT = """+# comment reference: context.parameters['ignored']+await page.fill(value=context.parameters['real'])+"""+++def test_guard_passes_when_all_refs_declared() -> None:+ result = validate_context_parameter_refs(+ code=SCRIPT_WITH_TWO_REFS,+ declared_param_keys=frozenset({"search_term", "other_key"}),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ assert result.valid+ assert result.undeclared_refs == []+++def test_guard_accepts_via_synthesized_keys() -> None:+ """Phase 1: synthesized `GeneratedWorkflowParameters` fields count as valid."""+ result = validate_context_parameter_refs(+ code=SCRIPT_WITH_PHANTOM,+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset({"preprint_search_term"}),+ )+ assert result.valid+++def test_guard_accepts_via_upstream_schema() -> None:+ code = "value=context.parameters['invoice_date']"+ result = validate_context_parameter_refs(+ code=code,+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset({"invoice_date"}),+ synthesized_keys=frozenset(),+ )+ assert result.valid+++def test_guard_detects_phantom_param_when_nothing_covers_it() -> None:+ result = validate_context_parameter_refs(+ code=SCRIPT_WITH_PHANTOM,+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ assert not result.valid+ assert len(result.undeclared_refs) == 1+ assert result.undeclared_refs[0].key == "preprint_search_term"+++def test_guard_skips_references_in_comments() -> None:+ result = validate_context_parameter_refs(+ code=SCRIPT_WITH_COMMENT,+ declared_param_keys=frozenset({"real"}),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ assert result.valid, "commented-out reference must not count"+++def test_guard_collects_multiple_undeclared_refs() -> None:+ code = """+ await page.fill(value=context.parameters['a'])+ await page.fill(value=context.parameters['b'])+ await page.fill(value=context.parameters['c'])+ """+ result = validate_context_parameter_refs(+ code=code,+ declared_param_keys=frozenset({"a"}),+ upstream_schema_keys=frozenset({"b"}),+ synthesized_keys=frozenset(),+ )+ assert not result.valid+ assert [r.key for r in result.undeclared_refs] == ["c"]+++def test_guard_catches_get_access_form() -> None:+ """Regex matches both subscript and `.get()` access patterns."""+ code = """+ a = context.parameters.get('ok_key')+ b = context.parameters.get("with_default", "fallback")+ c = context.parameters['also_ok']+ d = context.parameters.get('phantom')+ """+ result = validate_context_parameter_refs(+ code=code,+ declared_param_keys=frozenset({"ok_key", "with_default", "also_ok"}),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ assert not result.valid+ assert [r.key for r in result.undeclared_refs] == ["phantom"]+++def test_guard_handles_double_and_single_quotes() -> None:+ code = """+ x = context.parameters["double"]+ y = context.parameters['single']+ """+ result = validate_context_parameter_refs(+ code=code,+ declared_param_keys=frozenset({"double", "single"}),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ assert result.valid+++def test_format_error_contains_invalid_and_valid_keys() -> None:+ result = validate_context_parameter_refs(+ code="context.parameters['phantom']",+ declared_param_keys=frozenset({"real_key"}),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ msg = result.format_error()+ assert "'phantom'" in msg+ assert "'real_key'" in msg+ assert "SKY-8965" in msg+++# --- log_or_raise_guard_result --------------------------------------------+++def test_log_or_raise_noop_on_valid_result() -> None:+ result = validate_context_parameter_refs(+ code="no refs here",+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ # Should not raise even with raise_on_violation=True+ log_or_raise_guard_result(result, raise_on_violation=True)+++def test_log_or_raise_does_not_raise_phase_1() -> None:+ """Phase 1 behaviour: raise_on_violation=False → log only.++ The log assertion is verified manually via Datadog in production — structlog+ doesn't propagate to pytest's caplog without extra fixture setup, so we+ assert the no-raise contract here and rely on integration / production+ observability for the log payload.+ """+ result = validate_context_parameter_refs(+ code="context.parameters['phantom']",+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ # Does not raise+ log_or_raise_guard_result(+ result,+ raise_on_violation=False,+ workflow_permanent_id="wpid_test",+ workflow_run_id="wr_test",+ )+++def test_log_or_raise_raises_phase_2() -> None:+ """Phase 2 behaviour: raise_on_violation=True → throws HallucinatedParameterError."""+ result = validate_context_parameter_refs(+ code="context.parameters['phantom']",+ declared_param_keys=frozenset(),+ upstream_schema_keys=frozenset(),+ synthesized_keys=frozenset(),+ )+ with pytest.raises(HallucinatedParameterError) as exc_info:+ log_or_raise_guard_result(result, raise_on_violation=True)+ assert "'phantom'" in str(exc_info.value)+ assert exc_info.value.result is result+++def test_collect_declared_params_includes_all_parameter_types() -> None:+ workflow = {+ "workflow_definition": {+ "parameters": [+ {"parameter_type": "workflow", "key": "search_term"},+ {"parameter_type": "output", "key": "extracted_date"},+ {"parameter_type": "context", "key": "loop_var"},+ {"parameter_type": "aws_secret", "key": "api_token"},+ ]+ }+ }+ keys = _collect_declared_param_keys(workflow)+ assert keys == frozenset({"search_term", "extracted_date", "loop_var", "api_token"})+++def test_collect_declared_params_returns_empty_on_non_dict_definition() -> None:+ assert _collect_declared_param_keys({"workflow_definition": "not a dict"}) == frozenset()+ assert _collect_declared_param_keys({}) == frozenset()+++def test_collect_upstream_schema_keys_parses_json_string_schema() -> None:+ blocks = [{"data_schema": '{"properties": {"invoice_date": {"type": "string"}, "total": {"type": "number"}}}'}]+ keys = _collect_upstream_schema_keys(blocks)+ assert keys == frozenset({"invoice_date", "total"})+++def test_collect_upstream_schema_keys_ignores_invalid_json_string_schema() -> None:+ blocks = [{"data_schema": "this is not json"}]+ assert _collect_upstream_schema_keys(blocks) == frozenset()+++def test_collect_upstream_schema_keys_recurses_into_loop_blocks() -> None:+ blocks = [+ {+ "block_type": "for_loop",+ "loop_blocks": [+ {+ "data_schema": {"properties": {"nested_invoice_id": {"type": "string"}}},+ },+ ],+ },+ {"data_schema": {"properties": {"outer_key": {"type": "string"}}}},+ ]+ keys = _collect_upstream_schema_keys(blocks)+ assert keys == frozenset({"nested_invoice_id", "outer_key"})