Skip to content

Automate browser based workflows with AI

Revert #10280 — accidental mass deletion of copilot code (#5525)

Andrew Neilsoncf4d378a67db
diff --git a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
index dac552fe8..ba7abe404 100644
--- a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
+++ b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
@@ -307,6 +307,14 @@ function RunWorkflowForm({
);
const hasLoginBlockValidationError = loginBlocksWithoutCredentials.length > 0;
+ const blockingParameterTypes = new Set([
+ "boolean",
+ "integer",
+ "float",
+ "file_url",
+ "json",
+ ]);
+
const form = useForm<RunWorkflowFormType>({
mode: "onTouched",
reValidateMode: "onChange",
@@ -325,6 +333,13 @@ function RunWorkflowForm({
},
});
+ const formErrors = form.formState.errors;
+ const hasBlockingParameterError = workflowParameters.some(
+ (param) =>
+ blockingParameterTypes.has(param.workflow_parameter_type) &&
+ formErrors[param.key],
+ );
+
const runWorkflowMutation = useMutation({
mutationFn: async (values: RunWorkflowFormType) => {
const client = await getClient(credentialGetter);
@@ -497,11 +512,7 @@ function RunWorkflowForm({
const handleInvalid = (errors: FieldErrors<RunWorkflowFormType>) => {
const hasBlockingErrors = workflowParameters.some(
(param) =>
- (param.workflow_parameter_type === "boolean" ||
- param.workflow_parameter_type === "integer" ||
- param.workflow_parameter_type === "float" ||
- param.workflow_parameter_type === "file_url" ||
- param.workflow_parameter_type === "json") &&
+ blockingParameterTypes.has(param.workflow_parameter_type) &&
errors[param.key],
);
@@ -560,7 +571,9 @@ function RunWorkflowForm({
<Button
type="submit"
disabled={
- runWorkflowMutation.isPending || hasLoginBlockValidationError
+ runWorkflowMutation.isPending ||
+ hasLoginBlockValidationError ||
+ hasBlockingParameterError
}
>
{runWorkflowMutation.isPending && (
diff --git a/skyvern-frontend/src/routes/workflows/components/CodeEditor.tsx b/skyvern-frontend/src/routes/workflows/components/CodeEditor.tsx
index ff18661a3..9c31bcda0 100644
--- a/skyvern-frontend/src/routes/workflows/components/CodeEditor.tsx
+++ b/skyvern-frontend/src/routes/workflows/components/CodeEditor.tsx
@@ -61,9 +61,11 @@ function CodeEditor({
}: Props) {
const viewRef = useRef<EditorView | null>(null);
const [internalValue, setInternalValue] = useState(value);
+ const latestValueRef = useRef(value);
useEffect(() => {
setInternalValue(value);
+ latestValueRef.current = value;
}, [value]);
const debouncedOnChange = useDebouncedCallback((newValue: string) => {
@@ -72,6 +74,7 @@ function CodeEditor({
const handleChange = (newValue: string) => {
setInternalValue(newValue);
+ latestValueRef.current = newValue;
debouncedOnChange(newValue);
};
@@ -159,7 +162,8 @@ function CodeEditor({
if (!viewRef.current) viewRef.current = viewUpdate.view;
}}
onBlur={() => {
- debouncedOnChange.flush();
+ debouncedOnChange.cancel();
+ onChange?.(latestValueRef.current);
}}
/>
);
diff --git a/skyvern-frontend/src/routes/workflows/components/scheduleParameters.ts b/skyvern-frontend/src/routes/workflows/components/scheduleParameters.ts
index f39472501..70f8aedbb 100644
--- a/skyvern-frontend/src/routes/workflows/components/scheduleParameters.ts
+++ b/skyvern-frontend/src/routes/workflows/components/scheduleParameters.ts
@@ -50,10 +50,21 @@ export function validateScheduleParameters(
const errors: Record<string, string> = {};
for (const parameter of parameters) {
if (!isScheduleParameter(parameter)) continue;
- if (!isRequired(parameter)) continue;
const value = values[parameter.key];
- if (isMissingRequiredValue(parameter, value)) {
+ if (isRequired(parameter) && isMissingRequiredValue(parameter, value)) {
errors[parameter.key] = "Required";
+ continue;
+ }
+ if (
+ parameter.workflow_parameter_type === "json" &&
+ typeof value === "string" &&
+ value.trim() !== ""
+ ) {
+ try {
+ JSON.parse(value);
+ } catch {
+ errors[parameter.key] = "Invalid JSON";
+ }
}
}
return errors;
diff --git a/skyvern/core/script_generations/deterministic_field_naming.py b/skyvern/core/script_generations/deterministic_field_naming.py
new file mode 100644
index 000000000..197645efe
--- /dev/null
+++ b/skyvern/core/script_generations/deterministic_field_naming.py
@@ -0,0 +1,231 @@
+"""Deterministic field-name picker for workflow-parameter synthesis.
+
+This is the replacement for `_generate_field_names_with_llm` in
+`generate_workflow_parameters.py`. In Phase 1 it ships as a pure,
+side-effect-free function that is NOT yet wired into the hot path — the
+LLM call remains the source of truth until the validation guard in
+`generate_script.py` gives us the production signal to flip the switch.
+
+The picker implements three rules in priority order for every INPUT_TEXT,
+UPLOAD_FILE, and SELECT_OPTION action observed during a workflow run. See
+SKY-8965 for the motivating smoke-test repro (phantom
+`preprint_search_term` on a single-block search workflow whose navigation
+goal embedded the search term as a literal).
+
+Rule precedence:
+ 1. Jinja-reference rule — the unrendered `navigation_goal` template
+ contains `{{ key }}` where `key` is in the valid-keys set
+ (declared workflow parameters ∪ upstream block schema keys).
+ Use `key` directly; no synthesis.
+ 2. Upstream-schema rule — the action's value equals a literal value
+ associated with an upstream block's `data_schema.properties` key.
+ Use that key; no synthesis.
+ 3. Intention-derived rule — deterministic snake_case sanitization of
+ the action's `intention` text. Last-resort synthesis; guaranteed
+ deterministic across runs with the same input.
+
+Rules 1 and 2 both produce field names that are already in the valid-keys
+set and therefore will never trip the validation guard. Rule 3 produces
+synthesized names that the guard will eventually reject in Phase 2, forcing
+workflow authors to declare their parameters up front or accept an
+ai='proactive' fallback in the script.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from skyvern.webeye.actions.actions import ActionType
+
+CUSTOM_FIELD_ACTIONS: tuple[ActionType, ...] = (
+ ActionType.INPUT_TEXT,
+ ActionType.UPLOAD_FILE,
+ ActionType.SELECT_OPTION,
+)
+
+
+@dataclass(frozen=True)
+class FieldPick:
+ """Result of picking a field strategy for a single action.
+
+ `rule` is the rule that fired: "jinja_ref" | "upstream_schema" |
+ "intention_derived". The caller can use this to emit telemetry on
+ rule-distribution and to branch in Phase 2 (e.g. Rule 3 becomes
+ inline-literal vs ai-proactive instead of synthesis).
+ """
+
+ field_name: str
+ rule: str
+ description: str | None = None
+
+
+# `{{ name }}` or `{{ name.attr }}` or `{{ name | filter }}` — we only
+# care about the root identifier.
+_JINJA_ROOT_RE = re.compile(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\b")
+
+# Sanitize an intention string down to a valid snake_case identifier.
+_NON_IDENT = re.compile(r"[^a-z0-9_]+")
+_REPEAT_UNDERSCORE = re.compile(r"_+")
+
+
+def sanitize_intention_to_field_name(intention: str, fallback: str = "unknown_field") -> str:
+ """Deterministic sanitization of an intention string to a Python identifier.
+
+ Mirrors the fallback pattern commented out in
+ `hydrate_input_text_actions_with_field_names`, now promoted to first-class.
+ """
+ if not intention:
+ return fallback
+ lowered = intention.lower().strip()
+ # Replace anything that isn't a-z0-9_ with underscores
+ cleaned = _NON_IDENT.sub("_", lowered)
+ # Collapse repeated underscores, strip leading/trailing
+ cleaned = _REPEAT_UNDERSCORE.sub("_", cleaned).strip("_")
+ if not cleaned:
+ return fallback
+ # Ensure it doesn't start with a digit
+ if cleaned[0].isdigit():
+ cleaned = f"f_{cleaned}"
+ # Cap length — intentions can be paragraphs
+ return cleaned[:60]
+
+
+def extract_jinja_root_names(template: str) -> set[str]:
+ """Return the set of root identifiers referenced in `{{ ... }}` blocks.
+
+ Example:
+ >>> extract_jinja_root_names("Search for {{ query }} then {{ user.name }}")
+ {"query", "user"}
+ """
+ if not template:
+ return set()
+ return set(_JINJA_ROOT_RE.findall(template))
+
+
+def pick_field_name_for_action(
+ *,
+ action: dict[str, Any],
+ goal_template: str,
+ declared_param_keys: frozenset[str],
+ upstream_schema_keys: frozenset[str],
+ existing_field_name: str | None = None,
+) -> FieldPick:
+ """Pick a field name for a single custom-field action.
+
+ Args:
+ action: INPUT_TEXT / UPLOAD_FILE / SELECT_OPTION action dict. Must
+ carry `intention` and (depending on action_type) `text` / `file_url` /
+ `option`.
+ goal_template: Unrendered `navigation_goal` string for the task this
+ action belongs to. Must NOT be the rendered form — otherwise the
+ jinja-reference rule cannot tell a real parameter from a literal.
+ declared_param_keys: Workflow-declared parameter keys
+ (`workflow.workflow_definition.parameters[].key`).
+ upstream_schema_keys: Keys collected from upstream blocks'
+ `data_schema.properties` (output-parameter keys are collected
+ separately via `_collect_declared_param_keys` at the workflow level).
+ existing_field_name: Pre-existing assignment that must be preserved
+ across regenerations (e.g., from a cached block's schema). When set,
+ it wins unconditionally — regenerations must not rename fields that
+ cached code references.
+
+ Returns:
+ FieldPick with the chosen name, the rule that fired, and (when the
+ picker used jinja/schema rules) an optional description for the
+ generated Pydantic field.
+
+ Never raises. The fallback path is guaranteed to produce a valid
+ identifier from the action's intention.
+ """
+ # Preservation wins: cached code already references this name.
+ if existing_field_name:
+ return FieldPick(field_name=existing_field_name, rule="existing_assignment")
+
+ valid_keys = declared_param_keys | upstream_schema_keys
+ referenced_keys = extract_jinja_root_names(goal_template)
+
+ # Rule 1: jinja reference to a declared or schema key.
+ intersection = referenced_keys & valid_keys
+ if intersection:
+ # When the goal references multiple keys we can't disambiguate here
+ # (we don't know which INPUT_TEXT action targets which field). Prefer
+ # a declared parameter over a schema key, then alphabetical for
+ # determinism.
+ # TODO(SKY-8965 Phase 2): before wiring this picker into the hot path,
+ # disambiguate multi-key goals by falling through to Rule 3 (or a new
+ # action-intention → key matcher) when `len(intersection) > 1`.
+ # Otherwise every INPUT_TEXT action in a multi-field form gets the same
+ # alphabetically-first name and the synthesized schema collapses fields.
+ declared_matches = intersection & declared_param_keys
+ preferred = sorted(declared_matches) if declared_matches else sorted(intersection)
+ return FieldPick(field_name=preferred[0], rule="jinja_ref")
+
+ # Rule 2: value matches an upstream schema key name or is clearly keyed
+ # by intention to one. Conservative check: if the intention text mentions
+ # a valid schema key (substring match, case-insensitive), bind to it.
+ intention = (action.get("intention") or "").lower()
+ for key in sorted(upstream_schema_keys):
+ if key.lower() in intention:
+ return FieldPick(field_name=key, rule="upstream_schema")
+
+ # Rule 3: deterministic fallback derived from intention.
+ name = sanitize_intention_to_field_name(intention)
+ description = action.get("intention") or f"Value for {name}"
+ return FieldPick(field_name=name, rule="intention_derived", description=description)
+
+
+def pick_field_names_for_actions(
+ *,
+ actions_by_task: dict[str, list[dict[str, Any]]],
+ goal_template_by_task: dict[str, str],
+ declared_param_keys: frozenset[str],
+ upstream_schema_keys: frozenset[str],
+ existing_field_assignments: dict[int, str] | None = None,
+) -> dict[str, FieldPick]:
+ """Bulk version: pick field names for every custom-field action.
+
+ Returns a mapping `"{task_id}:{action_id}" → FieldPick`. Actions without
+ a usable value are skipped — same behaviour as the existing LLM path in
+ `generate_workflow_parameters_schema`.
+
+ This function is pure. It does no I/O and makes no LLM calls.
+ """
+ existing_field_assignments = existing_field_assignments or {}
+ picks: dict[str, FieldPick] = {}
+ action_counter = 0
+
+ for task_id, actions in actions_by_task.items():
+ goal = goal_template_by_task.get(task_id, "")
+ for action in actions:
+ action_type = action.get("action_type", "")
+ if action_type not in CUSTOM_FIELD_ACTIONS:
+ continue
+
+ if action_type == ActionType.INPUT_TEXT:
+ value = action.get("text", "")
+ elif action_type == ActionType.UPLOAD_FILE:
+ value = action.get("file_url", "")
+ elif action_type == ActionType.SELECT_OPTION:
+ value = action.get("option", "")
+ else:
+ value = ""
+
+ if not value:
+ continue
+
+ action_counter += 1
+ existing = existing_field_assignments.get(action_counter)
+
+ pick = pick_field_name_for_action(
+ action=action,
+ goal_template=goal,
+ declared_param_keys=declared_param_keys,
+ upstream_schema_keys=upstream_schema_keys,
+ existing_field_name=existing,
+ )
+ key = f"{task_id}:{action.get('action_id', '')}"
+ picks[key] = pick
+
+ return picks
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.py
index 30578a71e..28681999b 100644
--- a/skyvern/core/script_generations/generate_script.py
+++ b/skyvern/core/script_generations/generate_script.py
@@ -7,6 +7,7 @@ Generate a runnable Skyvern workflow script.
from __future__ import annotations
import hashlib
+import json
import keyword
import re
from collections import deque
@@ -24,6 +25,11 @@ from skyvern.core.script_generations.generate_workflow_parameters import (
generate_workflow_parameters_schema,
hydrate_input_text_actions_with_field_names,
)
+from skyvern.core.script_generations.parameter_reference_guard import (
+ HallucinatedParameterError,
+ log_or_raise_guard_result,
+ validate_context_parameter_refs,
+)
from skyvern.forge import app
from skyvern.schemas.workflows import FileStorageType
from skyvern.utils.strings import sanitize_identifier
@@ -54,6 +60,77 @@ class CodeGenResult:
blocks_failed: int
+# ----- SKY-8965: valid-keys collection for the parameter-reference guard --------
+
+# `\s+` tolerates any indentation a future formatter might pick; `[^=]+?` for
+# the type annotation captures complex types like `Optional[str]` and `list[str]`
+# without silently dropping those fields from the valid-keys set.
+_SCHEMA_FIELD_RE = re.compile(r"^\s+(\w+)\s*:\s*[^=]+?=\s*Field\(", re.MULTILINE)
+
+
+def _collect_declared_param_keys(workflow: dict[str, Any]) -> frozenset[str]:
+ """Return the set of all parameter keys from the workflow definition.
+
+ Includes workflow, output, and context parameters — any parameter type whose
+ key can legally appear as `context.parameters['key']` at runtime. Secret
+ parameter types (aws_secret, bitwarden_*, etc.) are included too since their
+ presence in the valid-keys set only reduces false-positive guard violations.
+ """
+ keys: set[str] = set()
+ defn = workflow.get("workflow_definition") or {}
+ if not isinstance(defn, dict):
+ return frozenset()
+ for param in defn.get("parameters") or []:
+ if not isinstance(param, dict):
+ continue
+ key = param.get("key")
+ if key:
+ keys.add(key)
+ return frozenset(keys)
+
+
+def _collect_upstream_schema_keys(blocks: list[dict[str, Any]]) -> frozenset[str]:
+ """Return the set of keys introduced by upstream blocks' data_schema.
+
+ Extracts `properties` keys from any block's `data_schema` (extract-info
+ blocks and task blocks with a `data_extraction_goal`). Handles both dict
+ schemas and JSON-encoded string schemas (BlockYAML allows either). Recurses
+ into ForLoop children (`loop_blocks`) since extract-info nested inside a
+ loop still contributes keys referenced by the generated script.
+
+ Phase 1 treats these as globally valid (a reference to `invoice_date` is OK
+ anywhere in the script, not just blocks that come after the block that
+ declares it). Phase 2 may tighten this to per-block ordering.
+ """
+ keys: set[str] = set()
+ for block in blocks or []:
+ schema = block.get("data_schema")
+ if isinstance(schema, str):
+ try:
+ schema = json.loads(schema)
+ except (ValueError, TypeError):
+ schema = None
+ if isinstance(schema, dict):
+ props = schema.get("properties")
+ if isinstance(props, dict):
+ keys.update(str(k) for k in props.keys())
+ nested = block.get("loop_blocks")
+ if isinstance(nested, list):
+ keys |= _collect_upstream_schema_keys(nested)
+ return frozenset(keys)
+
+
+def _collect_synthesized_field_keys(schema_code: str) -> frozenset[str]:
+ """Extract field names from a generated `GeneratedWorkflowParameters` class.
+
+ Parses the schema code by regex for the `name: type = Field(...)` pattern.
+ Returns an empty frozenset for the empty-schema case (`pass` body).
+ """
+ if not schema_code:
+ return frozenset()
+ return frozenset(_SCHEMA_FIELD_RE.findall(schema_code))
+
+
def _build_existing_field_assignments(
blocks: list[dict[str, Any]],
actions_by_task: dict[str, list[dict[str, Any]]],
@@ -3366,7 +3443,36 @@ async def generate_workflow_script_python_code(
]
module = cst.Module(body=module_body)
- return CodeGenResult(source_code=module.code, blocks_created=blocks_created, blocks_failed=blocks_failed)
+ source_code = module.code
+
+ # SKY-8965 Phase 1: validate `context.parameters[X]` references against the
+ # expanded valid-keys set (declared params ∪ upstream block schema keys ∪
+ # synthesized `GeneratedWorkflowParameters` fields). Phase 1 logs violations
+ # only — we want production telemetry before raising in Phase 2. Wrapped in
+ # try/except so a guard bug can never break script generation.
+ try:
+ declared_keys = _collect_declared_param_keys(workflow)
+ upstream_keys = _collect_upstream_schema_keys(blocks)
+ synthesized_keys = _collect_synthesized_field_keys(generated_schema)
+ guard_result = validate_context_parameter_refs(
+ code=source_code,
+ declared_param_keys=declared_keys,
+ upstream_schema_keys=upstream_keys,
+ synthesized_keys=synthesized_keys,
+ )
+ log_or_raise_guard_result(
+ guard_result,
+ raise_on_violation=False,
+ workflow_permanent_id=workflow.get("workflow_permanent_id"),
+ workflow_run_id=run_id,
+ )
+ except HallucinatedParameterError:
+ # Phase 2 will raise this intentionally — don't swallow it.
+ raise
+ except Exception:
+ LOG.warning("parameter_reference_guard_failed_to_run", exc_info=True)
+
+ return CodeGenResult(source_code=source_code, blocks_created=blocks_created, blocks_failed=blocks_failed)
async def create_or_update_script_block(
diff --git a/skyvern/core/script_generations/parameter_reference_guard.py b/skyvern/core/script_generations/parameter_reference_guard.py
new file mode 100644
index 000000000..7f4faba49
--- /dev/null
+++ b/skyvern/core/script_generations/parameter_reference_guard.py
@@ -0,0 +1,167 @@
+"""Validation guard for undeclared `context.parameters[X]` references.
+
+Catches the phantom-parameter pattern at the end of initial script generation
+(SKY-8965). Complements `ScriptReviewer._validate_parameter_references`, which
+runs only during the reviewer retry path — this guard runs at the initial-gen
+pipeline stage, where the existing reviewer validator never sees the code.
+
+Phase 1 (this PR): guard runs in WARNING mode. It logs violations with full
+context but does not raise. This gives us a production signal in Datadog for
+hallucination frequency before we decide to escalate.
+
+Phase 2 (follow-up): flip the guard to raise on violation. At that point
+workflow authors either declare their parameters up front, or the script
+generator falls back to `ai='proactive'` for the undeclared fields (Phase 2
+change in `generate_workflow_parameters.py`).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+import structlog
+
+LOG = structlog.get_logger(__name__)
+
+
+# Matches both subscript (`context.parameters['key']`) and dict-get
+# (`context.parameters.get('key')` or `.get('key', default)`) access. The
+# current initial-gen emits subscripts only, but the reviewer retry path and
+# its prompt templates use `.get()`, so the guard handles both to stay
+# consistent with any future unification with ScriptReviewer._PARAM_REF_RE.
+# Attribute access (`context.parameters.key`) is NOT emitted anywhere today,
+# so it's intentionally excluded — add it if Phase 2 introduces it.
+_PARAM_REF_RE = re.compile(r"""context\.parameters(?:\[['"](\w+)['"]\]|\.get\(\s*['"](\w+)['"]\s*(?:,[^)]*)?\))""")
+
+
+@dataclass(frozen=True)
+class UndeclaredReference:
+ key: str
+ # One example location for debugging; we don't collect every occurrence.
+ example_line: str
+
+
+@dataclass
+class GuardResult:
+ """Result of validating a generated script against the valid-keys set."""
+
+ valid: bool
+ undeclared_refs: list[UndeclaredReference] = field(default_factory=list)
+ referenced_keys: frozenset[str] = frozenset()
+ valid_keys: frozenset[str] = frozenset()
+
+ def format_error(self) -> str:
+ """Human-readable error string for logs / raised exceptions."""
+ invalid_list = ", ".join(repr(r.key) for r in self.undeclared_refs)
+ valid_list = ", ".join(repr(k) for k in sorted(self.valid_keys)) or "(none)"
+ return (
+ f"Generated script references undeclared workflow parameters: {invalid_list}. "
+ f"Valid keys are: {valid_list}. This usually means the synthesis LLM invented a "
+ f"field name for a value that was a literal in the navigation goal. See SKY-8965."
+ )
+
+
+def _collect_refs(code: str) -> list[UndeclaredReference]:
+ """Walk the code line-by-line and collect every context.parameters[...] reference.
+
+ Comments are skipped. Multiple references per line produce multiple entries.
+ """
+ refs: list[UndeclaredReference] = []
+ for line in code.split("\n"):
+ stripped = line.lstrip()
+ if stripped.startswith("#"):
+ continue
+ for match in _PARAM_REF_RE.finditer(line):
+ # Group 1 = subscript form, group 2 = .get() form; exactly one fires.
+ key = match.group(1) or match.group(2)
+ if key:
+ refs.append(UndeclaredReference(key=key, example_line=line.strip()))
+ return refs
+
+
+def validate_context_parameter_refs(
+ *,
+ code: str,
+ declared_param_keys: frozenset[str],
+ upstream_schema_keys: frozenset[str],
+ synthesized_keys: frozenset[str] = frozenset(),
+) -> GuardResult:
+ """Validate that every `context.parameters[X]` in `code` references a known key.
+
+ The valid-keys set is the union of:
+ - `declared_param_keys`: every key declared in
+ `workflow_definition.parameters` regardless of parameter_type (workflow,
+ output, context, secret — all are legal `context.parameters[X]` targets
+ at runtime).
+ - `upstream_schema_keys`: keys exposed by upstream extract-info / task
+ blocks via their `data_schema.properties` (recursed into ForLoop
+ children).
+ - `synthesized_keys`: keys present in the generated `GeneratedWorkflowParameters`
+ Pydantic class. Callable from Phase 1 to stay compatible with the current
+ LLM synthesis path; Phase 2 will shrink this to empty and rely entirely
+ on the first two sets.
+
+ Args:
+ code: The Python source of the generated script (usually `main.py` +
+ concatenated block `.skyvern` files). Pass as a single string.
+ declared_param_keys: Frozenset of declared workflow parameter keys.
+ upstream_schema_keys: Frozenset of keys exposed by upstream blocks.
+ synthesized_keys: Frozenset of field names currently present in the
+ synthesized `GeneratedWorkflowParameters` class.
+
+ Returns:
+ A `GuardResult` with `valid=False` when any reference falls outside the
+ valid set. The caller decides whether to log (Phase 1) or raise
+ (Phase 2).
+ """
+ refs = _collect_refs(code)
+ referenced_keys = frozenset(r.key for r in refs)
+ valid_keys = declared_param_keys | upstream_schema_keys | synthesized_keys
+
+ undeclared = [r for r in refs if r.key not in valid_keys]
+ return GuardResult(
+ valid=not undeclared,
+ undeclared_refs=undeclared,
+ referenced_keys=referenced_keys,
+ valid_keys=valid_keys,
+ )
+
+
+class HallucinatedParameterError(RuntimeError):
+ """Raised in Phase 2 when the guard rejects the generated script.
+
+ Phase 1 does not raise; it only logs. The exception class is defined now
+ so downstream code can be written against it ahead of the flip.
+ """
+
+ def __init__(self, result: GuardResult) -> None:
+ super().__init__(result.format_error())
+ self.result = result
+
+
+def log_or_raise_guard_result(
+ result: GuardResult,
+ *,
+ raise_on_violation: bool,
+ workflow_permanent_id: str | None = None,
+ workflow_run_id: str | None = None,
+) -> None:
+ """Apply the Phase 1 / Phase 2 policy to a guard result.
+
+ Phase 1: `raise_on_violation=False`, we only log a warning.
+ Phase 2: `raise_on_violation=True`, we also raise `HallucinatedParameterError`.
+ """
+ if result.valid:
+ return
+
+ LOG.warning(
+ "parameter_reference_guard_violation",
+ workflow_permanent_id=workflow_permanent_id,
+ workflow_run_id=workflow_run_id,
+ undeclared_refs=sorted(r.key for r in result.undeclared_refs),
+ valid_keys=sorted(result.valid_keys),
+ sky_ticket="SKY-8965",
+ )
+ if raise_on_violation:
+ raise HallucinatedParameterError(result)
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index b1b6ed22c..c6904b8f0 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -5113,7 +5113,6 @@ class ForgeAgent:
summary_kwargs: dict[str, Any] = {
"data_extraction_goal": task.data_extraction_goal,
"data_extraction_schema": capped_schema,
- "current_url": scraped_page.url,
"local_datetime": local_datetime_str,
}
prompt = prompt_engine.load_prompt("data-extraction-summary", **summary_kwargs)
@@ -5124,25 +5123,27 @@ class ForgeAgent:
kwargs=summary_kwargs,
)
- # Cache the summary LLM call — the inputs (goal, schema, URL) are
- # identical across download-loop iterations that revisit the same page.
- # The `try` is narrowed to just compute_cache_key + lookup so a
- # downstream log failure can't double-count as a lookup_error.
+ # Cache the summary LLM call — the inputs (goal, schema) are identical
+ # across loop iterations that share an extraction configuration, even
+ # when each iteration navigates to a different URL. The `try` is
+ # narrowed to just compute_cache_key + lookup so a downstream log
+ # failure can't double-count as a lookup_error.
workflow_run_id = context.workflow_run_id if context else None
cache_key: str | None = None
lookup_result: extraction_cache.LookupResult | None = None
try:
- # data-extraction-summary is goal-agnostic: only goal/schema/URL/datetime
- # affect the rendered prompt, so everything else is intentionally omitted.
- # Hash the post-ceiling value for `data_extraction_schema` — when the
- # rendered prompt exceeds PROMPT_HARD_CEILING_TOKENS the ceiling drops
- # that field to None, so two oversized-schema requests that both get
- # dropped produce an identical LLM prompt and must share a cache key.
+ # data-extraction-summary plans *what* will be extracted from
+ # goal + schema; only those inputs, plus the date (via datetime),
+ # affect the rendered prompt, so everything else is intentionally
+ # omitted. Hash the post-ceiling value for `data_extraction_schema`
+ # — when the rendered prompt exceeds PROMPT_HARD_CEILING_TOKENS the
+ # ceiling drops that field to None, so two oversized-schema requests
+ # that both get dropped produce an identical LLM prompt and must
+ # share a cache key.
cache_key = extraction_cache.compute_cache_key(
call_path="agent",
data_extraction_goal=task.data_extraction_goal,
extracted_information_schema=post_ceiling_kwargs["data_extraction_schema"],
- current_url=scraped_page.url,
local_datetime=local_datetime_str,
llm_key=None,
)
diff --git a/skyvern/forge/prompts/skyvern/data-extraction-summary.j2 b/skyvern/forge/prompts/skyvern/data-extraction-summary.j2
index 55fc77921..a2773bf4d 100644
--- a/skyvern/forge/prompts/skyvern/data-extraction-summary.j2
+++ b/skyvern/forge/prompts/skyvern/data-extraction-summary.j2
@@ -5,8 +5,6 @@ Reply in JSON format with the following keys:
"summary": str, // Summary of the data you will extract within one sentence. Be precise and concise.
}
-The URL of the page you're on right now is `{{ current_url }}`.
-
Data extraction goal:
```
{{ data_extraction_goal }}
diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.py
new file mode 100644
index 000000000..be0fdc20f
--- /dev/null
+++ b/tests/unit/test_copilot_session_injection.py
@@ -0,0 +1,488 @@
+"""Tests for copilot session injection and output contract adapters."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.cli.core import session_manager
+from skyvern.cli.core.result import BrowserContext as MCPBrowserContext
+from skyvern.cli.core.session_manager import SessionState, scoped_session
+from skyvern.forge.sdk.copilot.runtime import AgentContext, mcp_to_copilot
+
+
[email protected](autouse=True)
+def _reset_session_state() -> None:
+ session_manager._current_session.set(None)
+ session_manager._global_session = None
+ session_manager.set_stateless_http_mode(False)
+
+
+def _make_stream() -> MagicMock:
+ stream = MagicMock()
+ stream.is_disconnected = AsyncMock(return_value=False)
+ return stream
+
+
+def _make_ctx(**overrides: Any) -> AgentContext:
+ defaults = dict(
+ organization_id="org-1",
+ workflow_id="wf-1",
+ workflow_permanent_id="wfp-1",
+ workflow_yaml="",
+ browser_session_id="pbs_test_123",
+ stream=_make_stream(),
+ api_key="sk-test-key",
+ )
+ defaults.update(overrides)
+ return AgentContext(**defaults)
+
+
+async def test_scoped_session_pushes_and_restores() -> None:
+ """scoped_session sets the ContextVar within scope and restores on exit."""
+ # ContextVar starts as None (from fixture reset)
+ assert session_manager._current_session.get() is None
+
+ injected = SessionState(
+ browser=MagicMock(),
+ context=MCPBrowserContext(mode="cloud_session", session_id="pbs_injected"),
+ )
+
+ async with scoped_session(injected):
+ inside = session_manager.get_current_session()
+ assert inside is injected
+ assert inside.context.session_id == "pbs_injected"
+
+ after = session_manager._current_session.get()
+ # Should be restored to None (the value before scoped_session set it)
+ assert after is None
+
+
+async def test_scoped_session_does_not_touch_global() -> None:
+ """scoped_session must NOT mutate _global_session."""
+ session_manager._global_session = None
+
+ injected = SessionState(
+ browser=MagicMock(),
+ context=MCPBrowserContext(mode="cloud_session", session_id="pbs_test"),
+ )
+
+ async with scoped_session(injected):
+ pass
+
+ assert session_manager._global_session is None
+
+
+async def test_scoped_session_concurrent_isolation() -> None:
+ """Two concurrent scoped_session calls don't interfere with each other."""
+ results: dict[str, str | None] = {}
+ both_entered = asyncio.Event()
+ entered_count = 0
+
+ async def worker(session_id: str) -> None:
+ nonlocal entered_count
+ state = SessionState(
+ browser=MagicMock(),
+ context=MCPBrowserContext(mode="cloud_session", session_id=session_id),
+ )
+ async with scoped_session(state):
+ entered_count += 1
+ if entered_count == 2:
+ both_entered.set()
+ # Wait until BOTH workers are inside their scope before reading —
+ # this guarantees ContextVar isolation is the only thing separating them.
+ await both_entered.wait()
+ current = session_manager.get_current_session()
+ results[session_id] = current.context.session_id if current.context else None
+
+ await asyncio.gather(worker("pbs_a"), worker("pbs_b"))
+
+ assert results["pbs_a"] == "pbs_a"
+ assert results["pbs_b"] == "pbs_b"
+ assert session_manager._global_session is None
+
+
+def test_mcp_to_copilot_success() -> None:
+ mcp_result = {
+ "ok": True,
+ "action": "skyvern_navigate",
+ "browser_context": {"mode": "cloud_session", "session_id": "pbs_1"},
+ "data": {"url": "https://example.com", "title": "Example"},
+ "timing_ms": {"total": 500},
+ "artifacts": [],
+ }
+ result = mcp_to_copilot(mcp_result)
+ assert result["ok"] is True
+ assert result["data"]["url"] == "https://example.com"
+ assert "action" not in result
+ assert "browser_context" not in result
+ assert "timing_ms" not in result
+ assert "artifacts" not in result
+
+
+def test_mcp_to_copilot_error() -> None:
+ mcp_result = {
+ "ok": False,
+ "error": {"code": "NO_ACTIVE_BROWSER", "message": "No browser", "hint": "Create one"},
+ }
+ result = mcp_to_copilot(mcp_result)
+ assert result["ok"] is False
+ assert "No browser" in result["error"]
+ assert "Create one" in result["error"]
+
+
+class TestMcpBrowserContextBridge:
+ """Bridge-specific behavior of mcp_browser_context (not scoped_session).
+
+ Covers: copilot session registry, API-key override install/reset, and the
+ teardown guarantees that must hold under every failure mode.
+ """
+
+ def _install_happy_path_mocks(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> tuple[MagicMock, MagicMock, MagicMock, MagicMock, list[Any]]:
+ import skyvern.forge.sdk.copilot.runtime as runtime
+
+ browser_state = MagicMock()
+ browser_state.browser_context = MagicMock()
+ manager = MagicMock()
+ manager.get_browser_state = AsyncMock(return_value=browser_state)
+ monkeypatch.setattr(runtime.app, "PERSISTENT_SESSIONS_MANAGER", manager)
+
+ monkeypatch.setattr(runtime, "get_skyvern", lambda: MagicMock())
+ monkeypatch.setattr(runtime, "SkyvernBrowser", lambda *a, **kw: MagicMock())
+ monkeypatch.setattr(runtime, "get_active_api_key", lambda: "sk-test-key")
+ monkeypatch.setattr(runtime, "hash_api_key_for_cache", lambda k: "hash_" + k)
+
+ override_token = object()
+ override_calls: list[Any] = []
+ monkeypatch.setattr(
+ runtime, "set_api_key_override", lambda k: (override_calls.append(("set", k)), override_token)[1]
+ )
+ monkeypatch.setattr(runtime, "reset_api_key_override", lambda t: override_calls.append(("reset", t)))
+
+ register_mock = MagicMock()
+ unregister_mock = MagicMock()
+ monkeypatch.setattr(runtime, "register_copilot_session", register_mock)
+ monkeypatch.setattr(runtime, "unregister_copilot_session", unregister_mock)
+
+ return manager, register_mock, unregister_mock, browser_state, override_calls
+
+ @pytest.mark.asyncio
+ async def test_happy_path_registers_session_and_balances_unregister(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
+
+ _, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)
+
+ ctx = _make_ctx()
+ async with mcp_browser_context(ctx):
+ # Inside the context, the bridge has registered a SessionState whose
+ # session_id matches the agent context — this is the public contract
+ # that resolve_browser(session_id=...) relies on.
+ args = register_mock.call_args.args
+ assert args[0] == ctx.browser_session_id
+ registered_state = args[1]
+ assert isinstance(registered_state, SessionState)
+ assert registered_state.context.session_id == ctx.browser_session_id
+
+ assert register_mock.call_count == 1
+ assert unregister_mock.call_count == 1
+ unregister_mock.assert_called_with(ctx.browser_session_id)
+ # Override installed then reset.
+ assert [c[0] for c in override_calls] == ["set", "reset"]
+
+ @pytest.mark.asyncio
+ async def test_missing_browser_context_raises_without_leaking_session_id(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ import skyvern.forge.sdk.copilot.runtime as runtime
+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
+
+ manager = MagicMock()
+ manager.get_browser_state = AsyncMock(return_value=None)
+ monkeypatch.setattr(runtime.app, "PERSISTENT_SESSIONS_MANAGER", manager)
+
+ ctx = _make_ctx()
+ with pytest.raises(RuntimeError, match="No browser context for copilot session") as exc_info:
+ async with mcp_browser_context(ctx):
+ pytest.fail("should not enter body")
+
+ # Session id must not leak into the user/LLM-visible exception message.
+ assert ctx.browser_session_id not in str(exc_info.value)
+
+ @pytest.mark.asyncio
+ async def test_exception_during_yield_still_tears_down(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
+
+ _, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)
+
+ class Boom(RuntimeError):
+ pass
+
+ ctx = _make_ctx()
+ with pytest.raises(Boom):
+ async with mcp_browser_context(ctx):
+ raise Boom("caller raised inside context")
+
+ # Both teardown paths must fire even when the caller raises.
+ assert register_mock.call_count == 1
+ assert unregister_mock.call_count == 1
+ assert [c[0] for c in override_calls] == ["set", "reset"]
+
+ @pytest.mark.asyncio
+ async def test_setup_phase_failure_still_resets_api_key_override(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """If get_skyvern raises AFTER set_api_key_override, the override must
+ still be reset so the request-scoped API key does not leak across
+ requests."""
+ import skyvern.forge.sdk.copilot.runtime as runtime
+ from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
+
+ _, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)
+
+ def _raising_get_skyvern() -> Any:
+ raise RuntimeError("skyvern client unavailable")
+
+ monkeypatch.setattr(runtime, "get_skyvern", _raising_get_skyvern)
+
+ ctx = _make_ctx()
+ with pytest.raises(RuntimeError, match="skyvern client unavailable"):
+ async with mcp_browser_context(ctx):
+ pytest.fail("should not enter body")
+
+ # Registration never happened because setup failed before register_copilot_session.
+ assert register_mock.call_count == 0
+ assert unregister_mock.call_count == 0
+ # But the override must have been set AND reset.
+ assert [c[0] for c in override_calls] == ["set", "reset"]
+
+
+class TestScreenshotAdapter:
+ @pytest.mark.asyncio
+ async def test_screenshot_post_hook_reshapes_data_with_url_and_title(self) -> None:
+ from skyvern.forge.sdk.copilot.tools import _screenshot_post_hook
+
+ ctx = _make_ctx()
+ raw = {"browser_context": {"url": "https://example.com", "title": "Example"}}
+ result = {
+ "ok": True,
+ "data": {"data": "iVBOR...", "mime": "image/png", "bytes": 1234},
+ }
+
+ adapted = await _screenshot_post_hook(result, raw, ctx)
+
+ assert adapted["data"]["screenshot_base64"] == "iVBOR..."
+ assert adapted["data"]["url"] == "https://example.com"
+ assert adapted["data"]["title"] == "Example"
+
+
+class TestNavigateAdapter:
+ @pytest.mark.asyncio
+ async def test_navigate_post_hook_lifts_url_and_adds_next_step(self) -> None:
+ from skyvern.forge.sdk.copilot.tools import _navigate_post_hook
+
+ ctx = _make_ctx()
+ raw = {"browser_context": {"url": "https://example.com", "title": "Example"}}
+ result = {"ok": True, "data": {"url": "https://example.com", "title": "Example"}}
+
+ adapted = await _navigate_post_hook(result, raw, ctx)
+
+ assert adapted["ok"] is True
+ assert adapted["url"] == "https://example.com"
+ assert "next_step" in adapted
+ assert "data" not in adapted
+
+
+class TestClickAdapter:
+ @pytest.mark.asyncio
+ async def test_click_post_hook_reshapes_data_with_url_and_title(self) -> None:
+ from skyvern.forge.sdk.copilot.tools import _click_post_hook
+
+ ctx = _make_ctx()
+ raw = {"browser_context": {"url": "https://ex.com", "title": "Page"}}
+ result = {
+ "ok": True,
+ "data": {"selector": "#btn", "intent": None, "sdk_equivalent": "..."},
+ }
+
+ adapted = await _click_post_hook(result, raw, ctx)
+
+ assert adapted["data"]["selector"] == "#btn"
+ assert adapted["data"]["url"] == "https://ex.com"
+ assert adapted["data"]["title"] == "Page"
+ assert "sdk_equivalent" not in adapted["data"]
+
+
+class TestTypeTextAdapter:
+ @pytest.mark.asyncio
+ async def test_type_text_post_hook_renames_text_length_to_typed_length(self) -> None:
+ from skyvern.forge.sdk.copilot.tools import _type_text_post_hook
+
+ ctx = _make_ctx()
+ raw = {"browser_context": {"url": "https://ex.com"}}
+ result = {
+ "ok": True,
+ "data": {"selector": "#email", "text_length": 15, "sdk_equivalent": "..."},
+ }
+
+ adapted = await _type_text_post_hook(result, raw, ctx)
+
+ assert adapted["data"]["selector"] == "#email"
+ assert adapted["data"]["typed_length"] == 15
+ assert adapted["data"]["url"] == "https://ex.com"
+ assert "text_length" not in adapted["data"]
+
+
+class TestEvaluateAdapter:
+ def test_evaluate_copilot_contract(self) -> None:
+ mcp_result = {
+ "ok": True,
+ "action": "skyvern_evaluate",
+ "data": {
+ "result": {"title": "Test"},
+ "sdk_equivalent": "await page.evaluate(...)",
+ },
+ "browser_context": {"mode": "cloud_session"},
+ }
+ result = mcp_to_copilot(mcp_result)
+ assert result["data"]["result"] == {"title": "Test"}
+
+
+class TestUpdateWorkflowDirect:
+ @pytest.mark.asyncio
+ async def test_calls_internal_with_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """update_workflow uses direct path even when api_key is set."""
+ from skyvern.forge.sdk.copilot.tools import _update_workflow
+
+ ctx = _make_ctx(api_key="sk-test-key", workflow_permanent_id="wpid_abc123")
+
+ mock_workflow = MagicMock()
+ mock_workflow.title = "Test"
+ mock_workflow.description = ""
+ mock_workflow.workflow_definition = MagicMock()
+ mock_workflow.workflow_definition.blocks = [MagicMock(), MagicMock()]
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ lambda **kwargs: mock_workflow,
+ )
+
+ mock_wf_service = MagicMock()
+ mock_wf_service.update_workflow_definition = AsyncMock()
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.app.WORKFLOW_SERVICE", mock_wf_service)
+
+ yaml_str = "title: Test\nworkflow_definition:\n blocks: []"
+ result = await _update_workflow({"workflow_yaml": yaml_str}, ctx)
+
+ assert result["ok"] is True
+ assert result["data"]["block_count"] == 2
+ assert result["_workflow"] is mock_workflow
+ assert ctx.workflow_yaml == yaml_str
+ mock_wf_service.update_workflow_definition.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_calls_internal_without_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """update_workflow uses direct path when api_key is None."""
+ from skyvern.forge.sdk.copilot.tools import _update_workflow
+
+ ctx = _make_ctx(api_key=None)
+
+ mock_workflow = MagicMock()
+ mock_workflow.title = "Test"
+ mock_workflow.description = ""
+ mock_workflow.workflow_definition = MagicMock()
+ mock_workflow.workflow_definition.blocks = []
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ lambda **kwargs: mock_workflow,
+ )
+
+ mock_wf_service = MagicMock()
+ mock_wf_service.update_workflow_definition = AsyncMock()
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.app.WORKFLOW_SERVICE", mock_wf_service)
+
+ result = await _update_workflow({"workflow_yaml": "title: Test"}, ctx)
+
+ assert result["ok"] is True
+ mock_wf_service.update_workflow_definition.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_yaml_parse_error_returns_failure(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """YAML parse errors return ok: false."""
+ import yaml as _yaml
+
+ from skyvern.forge.sdk.copilot.tools import _update_workflow
+
+ ctx = _make_ctx(api_key="sk-test-key")
+
+ def raise_yaml_error(**kwargs: Any) -> None:
+ raise _yaml.YAMLError("bad yaml")
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ raise_yaml_error,
+ )
+
+ result = await _update_workflow({"workflow_yaml": "bad: yaml: {"}, ctx)
+
+ assert result["ok"] is False
+ assert "Workflow validation failed" in result["error"]
+
+ @pytest.mark.asyncio
+ async def test_ctx_workflow_yaml_not_updated_on_failure(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """ctx.workflow_yaml should NOT be updated when processing fails."""
+ from pydantic import ValidationError as _ValidationError
+
+ from skyvern.forge.sdk.copilot.tools import _update_workflow
+
+ ctx = _make_ctx(
+ api_key="sk-test-key",
+ workflow_yaml="original yaml",
+ )
+
+ def raise_validation_error(**kwargs: Any) -> None:
+ raise _ValidationError.from_exception_data(
+ title="WorkflowCreateYAMLRequest",
+ line_errors=[],
+ )
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
+ raise_validation_error,
+ )
+
+ await _update_workflow({"workflow_yaml": "new broken yaml"}, ctx)
+
+ assert ctx.workflow_yaml == "original yaml"
+
+
+class TestWorkflowUpdatePersistence:
+ def test_record_marks_persisted_even_when_workflow_has_zero_blocks(self) -> None:
+ from skyvern.forge.sdk.copilot.tools import _record_workflow_update_result
+
+ ctx = MagicMock()
+ ctx.workflow_yaml = "title: Empty workflow"
+ ctx.workflow_persisted = False
+
+ workflow = MagicMock()
+ workflow.workflow_definition = MagicMock()
+ workflow.workflow_definition.blocks = []
+
+ _record_workflow_update_result(
+ ctx,
+ {
+ "ok": True,
+ "_workflow": workflow,
+ "data": {"block_count": 0},
+ },
+ )
+
+ assert ctx.last_workflow is workflow
+ assert ctx.last_workflow_yaml == "title: Empty workflow"
+ assert ctx.workflow_persisted is True
diff --git a/tests/unit/test_deterministic_field_naming.py b/tests/unit/test_deterministic_field_naming.py
new file mode 100644
index 000000000..c41efe671
--- /dev/null
+++ b/tests/unit/test_deterministic_field_naming.py
@@ -0,0 +1,246 @@
+"""Tests for the deterministic field-name picker (SKY-8965 Phase 1).
+
+See `skyvern/core/script_generations/deterministic_field_naming.py` for the
+three rules under test:
+ 1. jinja_ref — `{{ key }}` in unrendered goal references a known key
+ 2. upstream_schema — intention text references an upstream schema key
+ 3. intention_derived — deterministic fallback from the action's intention
+"""
+
+from __future__ import annotations
+
+from skyvern.core.script_generations.deterministic_field_naming import (
+ extract_jinja_root_names,
+ pick_field_name_for_action,
+ pick_field_names_for_actions,
+ sanitize_intention_to_field_name,
+)
+from skyvern.webeye.actions.actions import ActionType
+
+# --- sanitize_intention_to_field_name --------------------------------------
+
+
+def test_sanitize_preserves_valid_identifiers() -> None:
+ assert sanitize_intention_to_field_name("worker_name") == "worker_name"
+
+
+def test_sanitize_lowercases_and_replaces_spaces() -> None:
+ assert sanitize_intention_to_field_name("Worker Name") == "worker_name"
+
+
+def test_sanitize_strips_punctuation() -> None:
+ assert sanitize_intention_to_field_name("What's your email?") == "what_s_your_email"
+
+
+def test_sanitize_collapses_repeats() -> None:
+ assert sanitize_intention_to_field_name("foo bar baz") == "foo_bar_baz"
+
+
+def test_sanitize_handles_leading_digit() -> None:
+ result = sanitize_intention_to_field_name("123 dollars")
+ assert result.startswith("f_") and "123" in result
+
+
+def test_sanitize_caps_length() -> None:
+ long = "a " * 100
+ assert len(sanitize_intention_to_field_name(long)) <= 60
+
+
+def test_sanitize_empty_returns_fallback() -> None:
+ assert sanitize_intention_to_field_name("") == "unknown_field"
+ assert sanitize_intention_to_field_name("!!!") == "unknown_field"
+
+
+# --- extract_jinja_root_names ----------------------------------------------
+
+
+def test_jinja_root_names_simple() -> None:
+ assert extract_jinja_root_names("Search for {{ query }}") == {"query"}
+
+
+def test_jinja_root_names_multiple() -> None:
+ names = extract_jinja_root_names("{{ a }} and {{ b }} with {{ a }}")
+ assert names == {"a", "b"}
+
+
+def test_jinja_root_names_attribute_access() -> None:
+ # Only root identifier, not the attribute
+ assert extract_jinja_root_names("hi {{ user.name }}") == {"user"}
+
+
+def test_jinja_root_names_with_filters() -> None:
+ assert extract_jinja_root_names("{{ value | upper }}") == {"value"}
+
+
+def test_jinja_root_names_no_interpolation() -> None:
+ assert extract_jinja_root_names("no jinja here") == set()
+
+
+# --- pick_field_name_for_action --------------------------------------------
+
+
+def _input_action(text: str, intention: str = "", action_id: str = "a1") -> dict:
+ return {
+ "action_type": ActionType.INPUT_TEXT,
+ "text": text,
+ "intention": intention,
+ "action_id": action_id,
+ }
+
+
+def test_rule_1_jinja_reference_to_declared_param() -> None:
+ """If the goal contains `{{ key }}` and `key` is a declared param, use it."""
+ pick = pick_field_name_for_action(
+ action=_input_action("quantum computing", intention="search term"),
+ goal_template="Search for preprints about {{ search_term }}.",
+ declared_param_keys=frozenset({"search_term"}),
+ upstream_schema_keys=frozenset(),
+ )
+ assert pick.field_name == "search_term"
+ assert pick.rule == "jinja_ref"
+
+
+def test_rule_1_prefers_declared_param_over_schema_key_on_tie() -> None:
+ """When both sets match, declared-param beats schema-key."""
+ pick = pick_field_name_for_action(
+ action=_input_action("x", intention="pick one"),
+ goal_template="use {{ name_declared }} and {{ name_schema }}",
+ declared_param_keys=frozenset({"name_declared"}),
+ upstream_schema_keys=frozenset({"name_schema"}),
+ )
+ assert pick.field_name == "name_declared"
+ assert pick.rule == "jinja_ref"
+
+
+def test_rule_1_ignores_jinja_refs_not_in_valid_keys() -> None:
+ """A `{{ made_up }}` in the goal where `made_up` isn't declared → fall through."""
+ pick = pick_field_name_for_action(
+ action=_input_action("quantum computing", intention="search term for preprints"),
+ goal_template="Search for {{ made_up_name }}",
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ )
+ # No match → Rule 3 fallback
+ assert pick.rule == "intention_derived"
+
+
+def test_rule_2_upstream_schema_via_intention_substring() -> None:
+ """Intention text mentions an upstream schema key → use it."""
+ pick = pick_field_name_for_action(
+ action=_input_action("2026-04-15", intention="fill the invoice_date field"),
+ goal_template="Download the invoice for the extracted date",
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset({"invoice_date"}),
+ )
+ assert pick.field_name == "invoice_date"
+ assert pick.rule == "upstream_schema"
+
+
+def test_rule_3_fallback_from_intention() -> None:
+ """No declared param, no schema match → sanitize intention."""
+ pick = pick_field_name_for_action(
+ action=_input_action("quantum computing", intention="Enter the search term for preprints"),
+ goal_template="Search for 'quantum computing' preprints.",
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ )
+ assert pick.rule == "intention_derived"
+ assert pick.field_name.startswith("enter_the_search_term")
+
+
+def test_existing_field_name_wins_unconditionally() -> None:
+ """A preserved assignment from a cached block beats every rule."""
+ pick = pick_field_name_for_action(
+ action=_input_action("X", intention="whatever"),
+ goal_template="{{ completely_different_param }}",
+ declared_param_keys=frozenset({"completely_different_param"}),
+ upstream_schema_keys=frozenset(),
+ existing_field_name="legacy_name",
+ )
+ assert pick.field_name == "legacy_name"
+ assert pick.rule == "existing_assignment"
+
+
+# --- pick_field_names_for_actions (bulk) -----------------------------------
+
+
+def test_bulk_picks_skip_empty_values() -> None:
+ actions = {
+ "t1": [
+ {
+ "action_type": ActionType.INPUT_TEXT,
+ "text": "",
+ "intention": "empty",
+ "action_id": "a1",
+ },
+ {
+ "action_type": ActionType.INPUT_TEXT,
+ "text": "real",
+ "intention": "real value",
+ "action_id": "a2",
+ },
+ ]
+ }
+ picks = pick_field_names_for_actions(
+ actions_by_task=actions,
+ goal_template_by_task={"t1": ""},
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ )
+ assert "t1:a1" not in picks
+ assert "t1:a2" in picks
+
+
+def test_bulk_skips_non_custom_field_actions() -> None:
+ actions = {
+ "t1": [
+ {
+ "action_type": ActionType.CLICK,
+ "action_id": "a1",
+ },
+ {
+ "action_type": ActionType.INPUT_TEXT,
+ "text": "foo",
+ "intention": "type foo",
+ "action_id": "a2",
+ },
+ ]
+ }
+ picks = pick_field_names_for_actions(
+ actions_by_task=actions,
+ goal_template_by_task={"t1": ""},
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ )
+ assert list(picks.keys()) == ["t1:a2"]
+
+
+def test_bulk_applies_existing_assignments_by_counter_order() -> None:
+ actions = {
+ "t1": [
+ {
+ "action_type": ActionType.INPUT_TEXT,
+ "text": "x",
+ "intention": "first",
+ "action_id": "a1",
+ },
+ {
+ "action_type": ActionType.INPUT_TEXT,
+ "text": "y",
+ "intention": "second",
+ "action_id": "a2",
+ },
+ ]
+ }
+ # 1-indexed counter — assignment for the second action
+ picks = pick_field_names_for_actions(
+ actions_by_task=actions,
+ goal_template_by_task={"t1": ""},
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ existing_field_assignments={2: "locked_name"},
+ )
+ assert picks["t1:a2"].field_name == "locked_name"
+ assert picks["t1:a2"].rule == "existing_assignment"
+ # First action unchanged
+ assert picks["t1:a1"].rule == "intention_derived"
diff --git a/tests/unit/test_parameter_reference_guard.py b/tests/unit/test_parameter_reference_guard.py
new file mode 100644
index 000000000..605cb383a
--- /dev/null
+++ b/tests/unit/test_parameter_reference_guard.py
@@ -0,0 +1,245 @@
+"""Tests for the context.parameters reference guard (SKY-8965 Phase 1)."""
+
+from __future__ import annotations
+
+import pytest
+
+from skyvern.core.script_generations.generate_script import (
+ _collect_declared_param_keys,
+ _collect_upstream_schema_keys,
+)
+from skyvern.core.script_generations.parameter_reference_guard import (
+ HallucinatedParameterError,
+ log_or_raise_guard_result,
+ validate_context_parameter_refs,
+)
+
+SCRIPT_WITH_TWO_REFS = """
+async def block_fn(page, context):
+ await page.fill(value=context.parameters['search_term'])
+ await page.fill(value=context.parameters['other_key'])
+"""
+
+SCRIPT_WITH_PHANTOM = """
+async def block_fn(page, context):
+ await page.fill(value=context.parameters['preprint_search_term'])
+"""
+
+SCRIPT_WITH_COMMENT = """
+# comment reference: context.parameters['ignored']
+await page.fill(value=context.parameters['real'])
+"""
+
+
+def test_guard_passes_when_all_refs_declared() -> None:
+ result = validate_context_parameter_refs(
+ code=SCRIPT_WITH_TWO_REFS,
+ declared_param_keys=frozenset({"search_term", "other_key"}),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ assert result.valid
+ assert result.undeclared_refs == []
+
+
+def test_guard_accepts_via_synthesized_keys() -> None:
+ """Phase 1: synthesized `GeneratedWorkflowParameters` fields count as valid."""
+ result = validate_context_parameter_refs(
+ code=SCRIPT_WITH_PHANTOM,
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset({"preprint_search_term"}),
+ )
+ assert result.valid
+
+
+def test_guard_accepts_via_upstream_schema() -> None:
+ code = "value=context.parameters['invoice_date']"
+ result = validate_context_parameter_refs(
+ code=code,
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset({"invoice_date"}),
+ synthesized_keys=frozenset(),
+ )
+ assert result.valid
+
+
+def test_guard_detects_phantom_param_when_nothing_covers_it() -> None:
+ result = validate_context_parameter_refs(
+ code=SCRIPT_WITH_PHANTOM,
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ assert not result.valid
+ assert len(result.undeclared_refs) == 1
+ assert result.undeclared_refs[0].key == "preprint_search_term"
+
+
+def test_guard_skips_references_in_comments() -> None:
+ result = validate_context_parameter_refs(
+ code=SCRIPT_WITH_COMMENT,
+ declared_param_keys=frozenset({"real"}),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ assert result.valid, "commented-out reference must not count"
+
+
+def test_guard_collects_multiple_undeclared_refs() -> None:
+ code = """
+ await page.fill(value=context.parameters['a'])
+ await page.fill(value=context.parameters['b'])
+ await page.fill(value=context.parameters['c'])
+ """
+ result = validate_context_parameter_refs(
+ code=code,
+ declared_param_keys=frozenset({"a"}),
+ upstream_schema_keys=frozenset({"b"}),
+ synthesized_keys=frozenset(),
+ )
+ assert not result.valid
+ assert [r.key for r in result.undeclared_refs] == ["c"]
+
+
+def test_guard_catches_get_access_form() -> None:
+ """Regex matches both subscript and `.get()` access patterns."""
+ code = """
+ a = context.parameters.get('ok_key')
+ b = context.parameters.get("with_default", "fallback")
+ c = context.parameters['also_ok']
+ d = context.parameters.get('phantom')
+ """
+ result = validate_context_parameter_refs(
+ code=code,
+ declared_param_keys=frozenset({"ok_key", "with_default", "also_ok"}),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ assert not result.valid
+ assert [r.key for r in result.undeclared_refs] == ["phantom"]
+
+
+def test_guard_handles_double_and_single_quotes() -> None:
+ code = """
+ x = context.parameters["double"]
+ y = context.parameters['single']
+ """
+ result = validate_context_parameter_refs(
+ code=code,
+ declared_param_keys=frozenset({"double", "single"}),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ assert result.valid
+
+
+def test_format_error_contains_invalid_and_valid_keys() -> None:
+ result = validate_context_parameter_refs(
+ code="context.parameters['phantom']",
+ declared_param_keys=frozenset({"real_key"}),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ msg = result.format_error()
+ assert "'phantom'" in msg
+ assert "'real_key'" in msg
+ assert "SKY-8965" in msg
+
+
+# --- log_or_raise_guard_result --------------------------------------------
+
+
+def test_log_or_raise_noop_on_valid_result() -> None:
+ result = validate_context_parameter_refs(
+ code="no refs here",
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ # Should not raise even with raise_on_violation=True
+ log_or_raise_guard_result(result, raise_on_violation=True)
+
+
+def test_log_or_raise_does_not_raise_phase_1() -> None:
+ """Phase 1 behaviour: raise_on_violation=False → log only.
+
+ The log assertion is verified manually via Datadog in production — structlog
+ doesn't propagate to pytest's caplog without extra fixture setup, so we
+ assert the no-raise contract here and rely on integration / production
+ observability for the log payload.
+ """
+ result = validate_context_parameter_refs(
+ code="context.parameters['phantom']",
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ # Does not raise
+ log_or_raise_guard_result(
+ result,
+ raise_on_violation=False,
+ workflow_permanent_id="wpid_test",
+ workflow_run_id="wr_test",
+ )
+
+
+def test_log_or_raise_raises_phase_2() -> None:
+ """Phase 2 behaviour: raise_on_violation=True → throws HallucinatedParameterError."""
+ result = validate_context_parameter_refs(
+ code="context.parameters['phantom']",
+ declared_param_keys=frozenset(),
+ upstream_schema_keys=frozenset(),
+ synthesized_keys=frozenset(),
+ )
+ with pytest.raises(HallucinatedParameterError) as exc_info:
+ log_or_raise_guard_result(result, raise_on_violation=True)
+ assert "'phantom'" in str(exc_info.value)
+ assert exc_info.value.result is result
+
+
+def test_collect_declared_params_includes_all_parameter_types() -> None:
+ workflow = {
+ "workflow_definition": {
+ "parameters": [
+ {"parameter_type": "workflow", "key": "search_term"},
+ {"parameter_type": "output", "key": "extracted_date"},
+ {"parameter_type": "context", "key": "loop_var"},
+ {"parameter_type": "aws_secret", "key": "api_token"},
+ ]
+ }
+ }
+ keys = _collect_declared_param_keys(workflow)
+ assert keys == frozenset({"search_term", "extracted_date", "loop_var", "api_token"})
+
+
+def test_collect_declared_params_returns_empty_on_non_dict_definition() -> None:
+ assert _collect_declared_param_keys({"workflow_definition": "not a dict"}) == frozenset()
+ assert _collect_declared_param_keys({}) == frozenset()
+
+
+def test_collect_upstream_schema_keys_parses_json_string_schema() -> None:
+ blocks = [{"data_schema": '{"properties": {"invoice_date": {"type": "string"}, "total": {"type": "number"}}}'}]
+ keys = _collect_upstream_schema_keys(blocks)
+ assert keys == frozenset({"invoice_date", "total"})
+
+
+def test_collect_upstream_schema_keys_ignores_invalid_json_string_schema() -> None:
+ blocks = [{"data_schema": "this is not json"}]
+ assert _collect_upstream_schema_keys(blocks) == frozenset()
+
+
+def test_collect_upstream_schema_keys_recurses_into_loop_blocks() -> None:
+ blocks = [
+ {
+ "block_type": "for_loop",
+ "loop_blocks": [
+ {
+ "data_schema": {"properties": {"nested_invoice_id": {"type": "string"}}},
+ },
+ ],
+ },
+ {"data_schema": {"properties": {"outer_key": {"type": "string"}}}},
+ ]
+ keys = _collect_upstream_schema_keys(blocks)
+ assert keys == frozenset({"nested_invoice_id", "outer_key"})