Skip to content

Automate browser based workflows with AI

SKY-9453 fix copilot reword-turn handling: test run, labels, goal wrap (#5791)

Andrew Neilson048dca71d112
diff --git a/alembic/versions/2026_05_04_2320-b19685399f97_credits_used_cached_credits_used.py b/alembic/versions/2026_05_04_2320-b19685399f97_credits_used_cached_credits_used.py
new file mode 100644
index 000000000..e137ea867
--- /dev/null
+++ b/alembic/versions/2026_05_04_2320-b19685399f97_credits_used_cached_credits_used.py
@@ -0,0 +1,33 @@
+"""credits_used & cached_credits_used
+
+Revision ID: b19685399f97
+Revises: 696a7bba1ce6
+Create Date: 2026-05-04 23:20:09.907428+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b19685399f97"
+down_revision: Union[str, None] = "696a7bba1ce6"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column("workflow_runs", sa.Column("credits_used", sa.Integer(), server_default="0", nullable=True))
+ op.add_column("workflow_runs", sa.Column("cached_credits_used", sa.Integer(), server_default="0", nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("workflow_runs", "cached_credits_used")
+ op.drop_column("workflow_runs", "credits_used")
+ # ### end Alembic commands ###
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
index 65d2c6864..828923893 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-agent.j2
@@ -105,6 +105,7 @@ When the user corrects, amends, or updates a previous request (e.g., "I meant X
1. Update the workflow definition to reflect the correction — call update_workflow with the modified YAML.
2. Do NOT just navigate the browser to the new URL. The user wants the WORKFLOW itself changed.
3. After updating, test the corrected workflow with run_blocks_and_collect_debug.
+4. Rename affected block labels and block titles whenever the correction changes the subject, URL, target action, or extracted data. Labels become output keys; stale labels mislead the agent's own future tool calls and Jinja templates that reference them. When you rename a label, update every related `next_block_label`, `block_labels` argument, and Jinja block reference.
DIAGNOSING OUTPUT PROBLEMS:
When the user reports a problem with workflow output (duplicates, missing data, wrong values, errors):
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index cefa0611c..b5e5d70c8 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -398,7 +398,9 @@ def _translate_to_agent_result(
from skyvern.forge.sdk.copilot.tools import (
_banned_block_reject_message,
_detect_new_banned_blocks,
+ _detect_stale_block_metadata,
_record_banned_block_reject_span,
+ _stale_block_metadata_message,
)
banned_items = _detect_new_banned_blocks(workflow_yaml, ctx.last_workflow_yaml)
@@ -406,6 +408,11 @@ def _translate_to_agent_result(
_record_banned_block_reject_span("replace_workflow_inline", banned_items)
user_response = f"{user_response}\n\n(Note: {_banned_block_reject_message(banned_items)})"
workflow_yaml = ""
+ stale_metadata = _detect_stale_block_metadata(workflow_yaml, ctx.last_workflow_yaml or ctx.workflow_yaml)
+ if stale_metadata:
+ user_response = f"{user_response}\n\n(Note: {_stale_block_metadata_message(stale_metadata)})"
+ ctx.last_test_ok = None
+ workflow_yaml = ""
if workflow_yaml:
if ctx.user_message:
workflow_yaml = wrap_block_goals(workflow_yaml, ctx.user_message)
diff --git a/skyvern/forge/sdk/copilot/block_goal_wrapping.py b/skyvern/forge/sdk/copilot/block_goal_wrapping.py
index d365ed9fc..a0d4bcb30 100644
--- a/skyvern/forge/sdk/copilot/block_goal_wrapping.py
+++ b/skyvern/forge/sdk/copilot/block_goal_wrapping.py
@@ -13,6 +13,7 @@ Without this wrap:
from __future__ import annotations
+import re
from typing import Any
import yaml
@@ -39,6 +40,19 @@ _WRAPPABLE_FIELDS: tuple[str, ...] = (
# hard-coded substring) keeps idempotency intact if the template's wording
# changes.
_WRAPPED_PREFIX = MINI_GOAL_TEMPLATE.partition("{mini_goal}")[0]
+_MINI_GOAL_HEADER = "Achieve the following mini goal and once it's achieved, complete:"
+_MAIN_GOAL_HEADER = (
+ "This mini goal is part of the big goal the user wants to achieve and use the big goal as context to achieve "
+ "the mini goal:"
+)
+# Tolerates the spaced-fence variant (`` ` ` ` ``) that the LLM sometimes
+# emits to avoid nested markdown fences inside an already-wrapped goal.
+_FENCE_RE = r"`\s*`\s*`"
+_WRAPPED_GOAL_RE = re.compile(
+ rf"^\s*{re.escape(_MINI_GOAL_HEADER)}\s*{_FENCE_RE}\s*(?P<mini_goal>.*?)\s*{_FENCE_RE}\s*"
+ rf"{re.escape(_MAIN_GOAL_HEADER)}\s*{_FENCE_RE}\s*(?P<main_goal>.*?)\s*{_FENCE_RE}\s*$",
+ re.DOTALL,
+)
def wrap_block_goals(workflow_yaml: str, user_message: str) -> str:
@@ -86,7 +100,15 @@ def _wrap_blocks_in_place(blocks: list[Any], user_message: str) -> bool:
continue
for field_name in _WRAPPABLE_FIELDS:
value = block.get(field_name)
- if isinstance(value, str) and value and _WRAPPED_PREFIX not in value:
+ if not isinstance(value, str) or not value:
+ continue
+ normalized_existing_goal = _normalize_existing_wrapped_goal(value)
+ if normalized_existing_goal is not None:
+ if normalized_existing_goal != value:
+ block[field_name] = normalized_existing_goal
+ mutated = True
+ continue
+ if _WRAPPED_PREFIX not in value:
block[field_name] = MINI_GOAL_TEMPLATE.format(
mini_goal=value,
main_goal=user_message,
@@ -96,3 +118,32 @@ def _wrap_blocks_in_place(blocks: list[Any], user_message: str) -> bool:
if isinstance(loop_blocks, list):
mutated = _wrap_blocks_in_place(loop_blocks, user_message) or mutated
return mutated
+
+
+def _normalize_existing_wrapped_goal(value: str) -> str | None:
+ parsed = _extract_wrapped_goal(value)
+ if parsed is None:
+ return None
+ mini_goal, main_goal = parsed
+ return MINI_GOAL_TEMPLATE.format(mini_goal=mini_goal, main_goal=main_goal)
+
+
+# Bounded peel count: a reword turn stacks one layer; anything deeper is
+# malformed and we'd rather stop than recurse on adversarial input.
+_MAX_GOAL_UNWRAP_DEPTH = 8
+
+
+def _extract_wrapped_goal(value: str) -> tuple[str, str] | None:
+ current = value
+ last: tuple[str, str] | None = None
+ for _ in range(_MAX_GOAL_UNWRAP_DEPTH):
+ match = _WRAPPED_GOAL_RE.match(current)
+ if not match:
+ return last
+ mini_goal = match.group("mini_goal").strip()
+ main_goal = match.group("main_goal").strip()
+ if not mini_goal or not main_goal:
+ return last
+ last = (mini_goal, main_goal)
+ current = mini_goal
+ return last
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index 0b40c4a60..f48d57d40 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -7,7 +7,7 @@ import base64
import json
import re
import time
-from collections import defaultdict
+from collections import Counter, defaultdict
from datetime import datetime
from typing import Any, Literal
@@ -15,6 +15,7 @@ import structlog
import yaml
from agents import function_tool
from agents.run_context import RunContextWrapper
+from jinja2.sandbox import SandboxedEnvironment
from pydantic import ValidationError
from skyvern.forge import app
@@ -47,6 +48,7 @@ from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
from skyvern.forge.sdk.workflow.models.parameter import (
+ RESERVED_PARAMETER_KEYS,
OutputParameter,
WorkflowParameter,
WorkflowParameterType,
@@ -517,6 +519,15 @@ def _parameter_binding_invariant_error(
async def _update_workflow(params: dict[str, Any], ctx: AgentContext) -> dict[str, Any]:
workflow_yaml = params["workflow_yaml"]
+ # Prefer the most-recent in-turn emission so cross-path flows (inline
+ # REPLACE_WORKFLOW followed by update_workflow) compare against what the
+ # LLM actually saw, not the turn-start persisted state.
+ last_yaml = getattr(ctx, "last_workflow_yaml", None)
+ prior_yaml = last_yaml if isinstance(last_yaml, str) and last_yaml else ctx.workflow_yaml
+ stale_metadata = _detect_stale_block_metadata(workflow_yaml, prior_yaml)
+ if stale_metadata:
+ return {"ok": False, "error": _stale_block_metadata_message(stale_metadata)}
+
# Post-emission reject of copilot-v2 writes that introduce a banned
# block type. The schema pre_hook only fires when the LLM consults the
# schema; this safety net fires regardless of emission path. Label-based
@@ -619,7 +630,34 @@ async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[s
# to an upstream state-establisher, or falls back to the full requested list.
_BLOCK_TYPES_STATE_ESTABLISHER = frozenset({"navigation", "login", "goto_url"})
-_OUTPUT_REF_RE = re.compile(r"\{\{\s*([A-Za-z0-9_]+)_output\s*[\.}|]")
+_JINJA_IDENTIFIER = r"[A-Za-z_][A-Za-z0-9_]*"
+_OUTPUT_REF_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})_output\s*(?=[\.|}}])")
+_BLOCK_FORM_REF_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})\s*\.")
+_JINJA_ROOT_RE = re.compile(rf"\{{\{{\s*({_JINJA_IDENTIFIER})\s*(?=[\.|}}])")
+
+_JINJA_RUNTIME_GLOBAL_ROOTS = frozenset(SandboxedEnvironment().globals)
+_JINJA_LITERAL_ROOTS = frozenset({"none", "true", "false"})
+_JINJA_SPECIAL_CONTEXT_ROOTS = frozenset({"loop", "self", "varargs", "kwargs"})
+_SKYVERN_TEMPLATE_CONTEXT_ROOTS = frozenset(RESERVED_PARAMETER_KEYS) | frozenset(
+ {
+ "parameters",
+ "browser_session_id",
+ "organization_id",
+ # Conditional / branch evaluation roots — see BranchEvaluationContext.build_template_data.
+ "params",
+ "outputs",
+ "environment",
+ "env",
+ "llm",
+ }
+)
+_TEMPLATE_BUILTIN_ROOTS = (
+ _JINJA_RUNTIME_GLOBAL_ROOTS | _JINJA_LITERAL_ROOTS | _JINJA_SPECIAL_CONTEXT_ROOTS | _SKYVERN_TEMPLATE_CONTEXT_ROOTS
+)
+
+# Keep this to grammatical glue only. Workflow/action words are intentionally
+# not filtered; the two-token stale threshold is the conservative guardrail.
+_BLOCK_METADATA_STOPWORDS = frozenset({"and", "for", "the", "with"})
def _block_type_name(block: Any) -> str:
@@ -643,6 +681,218 @@ def _blocks_by_label(workflow_definition: Any) -> dict[str, Any]:
return by_label
+# Minimum length to apply the trailing-``s`` plural strip; below this we
+# leave the token alone so words like ``is``/``us``/``has`` aren't mangled.
+_MIN_STEMMABLE_TOKEN_LEN = 5
+
+
+def _metadata_token(token: str) -> str:
+ token = token.lower()
+ if len(token) >= _MIN_STEMMABLE_TOKEN_LEN and token.endswith("s"):
+ token = token[:-1]
+ return token
+
+
+def _metadata_tokens(value: Any) -> set[str]:
+ if not isinstance(value, str):
+ return set()
+ tokens: set[str] = set()
+ for token in re.findall(r"[A-Za-z0-9]+", value):
+ normalized = _metadata_token(token)
+ if len(normalized) <= 2 or normalized in _BLOCK_METADATA_STOPWORDS:
+ continue
+ tokens.add(normalized)
+ return tokens
+
+
+def _iter_yaml_blocks(blocks: Any) -> list[dict[str, Any]]:
+ found: list[dict[str, Any]] = []
+ if not isinstance(blocks, list):
+ return found
+ for block in blocks:
+ if not isinstance(block, dict):
+ continue
+ found.append(block)
+ loop_blocks = block.get("loop_blocks")
+ if isinstance(loop_blocks, list):
+ found.extend(_iter_yaml_blocks(loop_blocks))
+ return found
+
+
+def _workflow_yaml_blocks_by_label(workflow_yaml: str | None) -> dict[str, dict[str, Any]]:
+ if not workflow_yaml:
+ return {}
+ try:
+ parsed = safe_load_no_dates(workflow_yaml)
+ except yaml.YAMLError:
+ return {}
+ if not isinstance(parsed, dict):
+ return {}
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, dict):
+ return {}
+ by_label: dict[str, dict[str, Any]] = {}
+ for block in _iter_yaml_blocks(workflow_definition.get("blocks")):
+ label = block.get("label")
+ if isinstance(label, str):
+ by_label[label] = block
+ return by_label
+
+
+def _semantic_tokens_from_yaml(value: Any, *, exclude_keys: frozenset[str]) -> set[str]:
+ tokens: set[str] = set()
+ if isinstance(value, str):
+ return _metadata_tokens(value)
+ if isinstance(value, list):
+ for item in value:
+ tokens.update(_semantic_tokens_from_yaml(item, exclude_keys=exclude_keys))
+ return tokens
+ if isinstance(value, dict):
+ for key, item in value.items():
+ if key in exclude_keys:
+ continue
+ tokens.update(_semantic_tokens_from_yaml(item, exclude_keys=exclude_keys))
+ return tokens
+
+
+def _stale_metadata_reason(
+ *,
+ field_name: str,
+ field_value: Any,
+ prior_block: dict[str, Any],
+ submitted_block: dict[str, Any],
+ current_exclude_keys: frozenset[str],
+) -> str | None:
+ tokens = _metadata_tokens(field_value)
+ if len(tokens) < 2:
+ return None
+
+ prior_tokens = _semantic_tokens_from_yaml(prior_block, exclude_keys=current_exclude_keys)
+ current_tokens = _semantic_tokens_from_yaml(submitted_block, exclude_keys=current_exclude_keys)
+ removed_tokens = prior_tokens - current_tokens
+ if len(tokens & removed_tokens) < 2:
+ return None
+
+ return f"{field_name} {field_value!r} appears stale"
+
+
+def _prior_blocks_by_unique_title(prior_by_label: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
+ titled = [(b["title"], b) for b in prior_by_label.values() if isinstance(b.get("title"), str) and b["title"]]
+ counts = Counter(title for title, _ in titled)
+ return {title: block for title, block in titled if counts[title] == 1}
+
+
+_STALE_BASE_EXCLUDE = frozenset({"label", "next_block_label"})
+_STALE_TITLE_EXCLUDE = frozenset({"label", "title", "next_block_label"})
+
+
+def _stale_for_renamed_label(
+ label: str,
+ submitted_block: dict[str, Any],
+ prior_by_unique_title: dict[str, dict[str, Any]],
+) -> dict[str, Any] | None:
+ submitted_title = submitted_block.get("title")
+ # Title-less blocks intentionally skip the renamed-label path: with no
+ # title there is no cross-reference to a prior block, so we'd be
+ # guessing whether the rename was warranted.
+ if not (isinstance(submitted_title, str) and submitted_title):
+ return None
+ prior_block = prior_by_unique_title.get(submitted_title)
+ if prior_block is None:
+ return None
+ title_reason = _stale_metadata_reason(
+ field_name="title",
+ field_value=submitted_title,
+ prior_block=prior_block,
+ submitted_block=submitted_block,
+ current_exclude_keys=_STALE_TITLE_EXCLUDE,
+ )
+ if not title_reason:
+ return None
+ return {"label": label, "reasons": [title_reason]}
+
+
+def _stale_for_matched_label(
+ label: str,
+ submitted_block: dict[str, Any],
+ prior_block: dict[str, Any],
+) -> dict[str, Any] | None:
+ reasons: list[str] = []
+ submitted_title = submitted_block.get("title")
+ prior_title = prior_block.get("title")
+ title_reason = None
+ if isinstance(submitted_title, str) and submitted_title and submitted_title == prior_title:
+ title_reason = _stale_metadata_reason(
+ field_name="title",
+ field_value=submitted_title,
+ prior_block=prior_block,
+ submitted_block=submitted_block,
+ current_exclude_keys=_STALE_TITLE_EXCLUDE,
+ )
+ if title_reason:
+ reasons.append(title_reason)
+
+ # When the title was already flagged stale, exclude its tokens from the
+ # label-stale comparison so the same words don't double-count; otherwise
+ # the title's tokens are stable content that should weigh in.
+ label_exclude_keys = _STALE_TITLE_EXCLUDE if title_reason else _STALE_BASE_EXCLUDE
+ label_reason = _stale_metadata_reason(
+ field_name="label",
+ field_value=label,
+ prior_block=prior_block,
+ submitted_block=submitted_block,
+ current_exclude_keys=label_exclude_keys,
+ )
+ if label_reason:
+ reasons.insert(0, label_reason)
+
+ if not reasons:
+ return None
+ return {"label": label, "reasons": reasons}
+
+
+def _detect_stale_block_metadata(submitted_yaml: str | None, prior_yaml: str | None) -> list[dict[str, Any]]:
+ """Find corrected blocks whose old label/title no longer matches their revised goal text."""
+ prior_by_label = _workflow_yaml_blocks_by_label(prior_yaml)
+ submitted_by_label = _workflow_yaml_blocks_by_label(submitted_yaml)
+ if not prior_by_label or not submitted_by_label:
+ return []
+
+ prior_by_unique_title = _prior_blocks_by_unique_title(prior_by_label)
+
+ stale_items: list[dict[str, Any]] = []
+ for label, submitted_block in submitted_by_label.items():
+ prior_block = prior_by_label.get(label)
+ if prior_block is None:
+ item = _stale_for_renamed_label(label, submitted_block, prior_by_unique_title)
+ else:
+ item = _stale_for_matched_label(label, submitted_block, prior_block)
+ if item is not None:
+ stale_items.append(item)
+ return stale_items
+
+
+_STALE_BLOCK_METADATA_MESSAGE_LIMIT = 5
+
+
+def _stale_block_metadata_message(items: list[dict[str, Any]]) -> str:
+ details = []
+ for item in items[:_STALE_BLOCK_METADATA_MESSAGE_LIMIT]:
+ label = item.get("label", "?")
+ reasons = item.get("reasons") or []
+ detail = "; ".join(str(reason) for reason in reasons)
+ details.append(f"{label}: {detail}")
+ if len(items) > _STALE_BLOCK_METADATA_MESSAGE_LIMIT:
+ details.append(f"(and {len(items) - _STALE_BLOCK_METADATA_MESSAGE_LIMIT} more)")
+ joined = "; ".join(details)
+ return (
+ "Workflow validation failed: corrected block metadata still appears stale. "
+ "When changing a user's requested subject, URL, or action, rename affected block labels and titles "
+ "to match the revised goal, and update next_block_label, block_labels, and Jinja references accordingly. "
+ f"Stale metadata: {joined}"
+ )
+
+
def _find_invalidated_labels(
old_definition: Any,
new_definition: Any,
@@ -698,20 +948,98 @@ def _nearest_upstream_state_establisher(
return None
-def _referenced_output_labels(frontier_labels: list[str], new_definition: Any) -> set[str]:
+def _serialized_frontier_block_configs(frontier_labels: list[str], new_definition: Any) -> list[str]:
by_label = _blocks_by_label(new_definition)
- needed: set[str] = set()
+ serialized_configs: list[str] = []
for label in frontier_labels:
block = by_label.get(label)
if block is None:
continue
try:
- serialized = json.dumps(_canonical_block_config(block), default=str, separators=(",", ":"))
+ serialized_configs.append(json.dumps(_canonical_block_config(block), default=str, separators=(",", ":")))
except (TypeError, ValueError):
- serialized = repr(block)
+ serialized_configs.append(repr(block))
+ return serialized_configs
+
+
+def _workflow_parameter_keys(definition: Any) -> set[str]:
+ parameters = getattr(definition, "parameters", None) if definition else None
+ keys: set[str] = set()
+ if not parameters:
+ return keys
+ for parameter in parameters:
+ key = getattr(parameter, "key", None)
+ if isinstance(key, str):
+ keys.add(key)
+ return keys
+
+
+_CREDENTIAL_REAL_VALUE_SUFFIXES = ("_real_username", "_real_password")
+
+
+def _classify_frontier_jinja_refs(
+ frontier_labels: list[str],
+ new_definition: Any,
+ serialized_configs: list[str] | None = None,
+) -> tuple[set[str], set[str], set[str]]:
+ """Single pass over frontier blocks; returns ``(suffix_form_refs, block_form_refs, unknown_roots)``."""
+ if serialized_configs is None:
+ serialized_configs = _serialized_frontier_block_configs(frontier_labels, new_definition)
+ known_labels = set(_blocks_by_label(new_definition))
+ parameter_keys = _workflow_parameter_keys(new_definition)
+ known_roots = known_labels | parameter_keys | _TEMPLATE_BUILTIN_ROOTS
+
+ suffix_form_refs: set[str] = set()
+ block_form_refs: set[str] = set()
+ unknown_roots: set[str] = set()
+
+ for serialized in serialized_configs:
for match in _OUTPUT_REF_RE.findall(serialized):
- needed.add(match)
- return needed
+ if match in known_labels:
+ suffix_form_refs.add(match)
+ for match in _BLOCK_FORM_REF_RE.findall(serialized):
+ if match in known_labels:
+ block_form_refs.add(match)
+ for root in _JINJA_ROOT_RE.findall(serialized):
+ if root in known_roots:
+ continue
+ if root.endswith("_output") and root[: -len("_output")] in known_labels:
+ continue
+ if any(
+ root.endswith(suffix) and root[: -len(suffix)] in parameter_keys
+ for suffix in _CREDENTIAL_REAL_VALUE_SUFFIXES
+ ):
+ continue
+ unknown_roots.add(root)
+
+ return suffix_form_refs, block_form_refs, unknown_roots
+
+
+def _referenced_output_labels(
+ frontier_labels: list[str],
+ new_definition: Any,
+ serialized_configs: list[str] | None = None,
+) -> set[str]:
+ suffix_refs, block_form_refs, _ = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
+ return suffix_refs | block_form_refs
+
+
+def _block_form_output_labels(
+ frontier_labels: list[str],
+ new_definition: Any,
+ serialized_configs: list[str] | None = None,
+) -> set[str]:
+ _, block_form_refs, _ = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
+ return block_form_refs
+
+
+def _unknown_jinja_roots(
+ frontier_labels: list[str],
+ new_definition: Any,
+ serialized_configs: list[str] | None = None,
+) -> set[str]:
+ _, _, unknown_roots = _classify_frontier_jinja_refs(frontier_labels, new_definition, serialized_configs)
+ return unknown_roots
def _summarize_action_trace(action_trace: list[dict[str, Any]] | None) -> list[str]:
@@ -846,20 +1174,24 @@ def _seed_for_frontier(
prefix_labels = requested_labels[:idx]
if not prefix_labels:
return labels_to_execute, {}, frontier
- # Only seed outputs that are actually referenced by the frontier suffix.
- # Over-seeding would weaken the "seed what downstream needs" discipline
- # and risks masking bugs where a block references an output that doesn't
- # flow through the normal {{label_output}} path.
- needed = _referenced_output_labels(labels_to_execute, new_definition)
- prefix_needed = [label for label in prefix_labels if label in needed]
+ serialized_configs = _serialized_frontier_block_configs(labels_to_execute, new_definition)
+ suffix_refs, block_form_refs, unknown_roots = _classify_frontier_jinja_refs(
+ labels_to_execute, new_definition, serialized_configs
+ )
+ if any(label in block_form_refs for label in prefix_labels):
+ # Seeded block_outputs only register <label>_output; block-form refs
+ # need a normal upstream execution to populate the <label> namespace.
+ return requested_labels, {}, requested_labels[0]
+ needed = suffix_refs | block_form_refs
seed: dict[str, Any] = {}
- for label in prefix_needed:
- if label in verified_outputs:
- seed[label] = verified_outputs[label]
- else:
- # A referenced output is missing from the verified cache — we
- # can't safely seed just the suffix. Fall back to the full list.
+ for label in prefix_labels:
+ if label not in needed:
+ continue
+ if label not in verified_outputs:
return requested_labels, {}, requested_labels[0]
+ seed[label] = verified_outputs[label]
+ if unknown_roots:
+ return requested_labels, {}, requested_labels[0]
return labels_to_execute, seed, frontier
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 0e6dbef10..1dc9f5b99 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -447,6 +447,9 @@ class WorkflowRunModel(Base):
)
copilot_session_id = Column(String, nullable=True)
+ credits_used = Column(Integer, nullable=True, default=0, server_default="0")
+ cached_credits_used = Column(Integer, nullable=True, default=0, server_default="0")
+
queued_at = Column(DateTime, nullable=True)
started_at = Column(DateTime, nullable=True)
finished_at = Column(DateTime, nullable=True)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 40211615f..59d6b50a5 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -297,6 +297,29 @@ class WorkflowRunsRepository(BaseRepository):
else:
raise WorkflowRunNotFound(workflow_run_id)
+ @db_operation("increment_workflow_run_credits")
+ async def increment_workflow_run_credits(
+ self,
+ workflow_run_id: str,
+ credits: int,
+ is_cached: bool = False,
+ ) -> None:
+ col = WorkflowRunModel.cached_credits_used if is_cached else WorkflowRunModel.credits_used
+ async with self.Session() as session:
+ result = await session.execute(
+ update(WorkflowRunModel)
+ .where(WorkflowRunModel.workflow_run_id == workflow_run_id)
+ .values({col: func.coalesce(col, 0) + credits})
+ )
+ if result.rowcount == 0:
+ LOG.warning(
+ "increment_workflow_run_credits matched no rows",
+ workflow_run_id=workflow_run_id,
+ credits=credits,
+ is_cached=is_cached,
+ )
+ await session.commit()
+
@db_operation("update_workflow_run_if_not_final")
async def update_workflow_run_if_not_final(
self,
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 7e259fbe5..4e785742e 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -507,6 +507,8 @@ def convert_to_workflow_run(
failure_category=workflow_run_model.failure_category,
ignore_inherited_workflow_system_prompt=workflow_run_model.ignore_inherited_workflow_system_prompt,
copilot_session_id=workflow_run_model.copilot_session_id,
+ credits_used=workflow_run_model.credits_used or 0,
+ cached_credits_used=workflow_run_model.cached_credits_used or 0,
)
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index 7d21f9aab..281d59e01 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -204,6 +204,8 @@ class WorkflowRun(BaseModel):
workflow_schedule_id: str | None = None
ignore_inherited_workflow_system_prompt: bool = False
copilot_session_id: str | None = None
+ credits_used: int = 0
+ cached_credits_used: int = 0
@field_validator("run_with", mode="before")
@classmethod
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 28fd53a27..e52026f4a 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -258,6 +258,46 @@ class TestTranslateToAgentResultGating:
assert agent_result.workflow_yaml is None
assert agent_result.response_type == "REPLACE_WORKFLOW"
+ def test_inline_replace_workflow_rejects_stale_block_metadata(self, monkeypatch) -> None:
+ # Inline REPLACE_WORKFLOW bypasses _update_workflow, so it must also
+ # reject a corrected workflow whose labels/titles still describe the
+ # prior subject.
+ process_mock = MagicMock(return_value=SimpleNamespace(name="new"))
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.tools._process_workflow_yaml", process_mock)
+
+ prior_yaml = """
+title: Count example.com topic alpha results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for topic alpha.
+"""
+ submitted_yaml = """
+title: Count example.com sample beta results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for sample beta.
+"""
+ ctx = _ctx(workflow_yaml=prior_yaml, last_workflow_yaml=prior_yaml, last_workflow=object(), last_test_ok=True)
+ result = _fake_run_result(
+ {"type": "REPLACE_WORKFLOW", "user_response": "Here you go.", "workflow_yaml": submitted_yaml}
+ )
+ agent_result = agent_module._translate_to_agent_result(
+ result, ctx, global_llm_context=None, chat_request=_chat_request(), organization_id="org-1"
+ )
+
+ process_mock.assert_not_called()
+ assert "corrected block metadata still appears stale" in agent_result.user_response
+ assert agent_result.updated_workflow is None
+ assert agent_result.workflow_yaml is None
+
def test_inline_replace_with_invalid_yaml_keeps_prior_pass(self, monkeypatch) -> None:
# _process_workflow_yaml raising on a malformed REPLACE must leave
# ctx untouched — no spurious last_test_ok reset, no workflow swap —
diff --git a/tests/unit/test_copilot_block_goal_wrapping.py b/tests/unit/test_copilot_block_goal_wrapping.py
index 01d269518..0a96fe1c6 100644
--- a/tests/unit/test_copilot_block_goal_wrapping.py
+++ b/tests/unit/test_copilot_block_goal_wrapping.py
@@ -143,6 +143,36 @@ def test_idempotent_on_already_wrapped_fields() -> None:
assert block["complete_criterion"] == already_wrapped_criterion
+def test_normalizes_spaced_fence_wrapped_goal_without_adding_reword_turn() -> None:
+ revised_goal = "Go to https://example.com. Tell me how many results there are referencing topic beta"
+ spaced_fence_wrapped_goal = textwrap.dedent(
+ f"""
+ Achieve the following mini goal and once it's achieved, complete:
+ ` ` `In the main search box, enter the phrase 'topic beta' and submit the search.` ` `
+
+ This mini goal is part of the big goal the user wants to achieve and use the big goal as context to achieve the mini goal:
+ ` ` `{revised_goal}` ` `
+ """
+ ).strip()
+ src = _yaml_with_blocks(
+ {
+ "block_type": "navigation",
+ "label": "search_topic_beta",
+ "navigation_goal": spaced_fence_wrapped_goal,
+ }
+ )
+
+ out = wrap_block_goals(src, "I meant topic beta")
+
+ goal = _blocks_from(out)[0]["navigation_goal"]
+ assert goal == MINI_GOAL_TEMPLATE.format(
+ mini_goal="In the main search box, enter the phrase 'topic beta' and submit the search.",
+ main_goal=revised_goal,
+ )
+ assert "I meant topic beta" not in goal
+ assert goal.count("Achieve the following mini goal") == 1
+
+
def test_noop_on_empty_user_message() -> None:
src = _yaml_with_blocks(
{
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index b5e714c09..f1e67c072 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -5,7 +5,9 @@ from __future__ import annotations
from typing import Any
import pytest
+from jinja2.sandbox import SandboxedEnvironment
+from skyvern.forge.sdk.copilot import tools
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
MAX_FAILED_TEST_NUDGES,
@@ -26,6 +28,7 @@ from skyvern.forge.sdk.copilot.tools import (
_plan_frontier,
_referenced_output_labels,
)
+from skyvern.forge.sdk.workflow.models.parameter import RESERVED_PARAMETER_KEYS
class _FakeBlock:
@@ -50,9 +53,15 @@ class _FakeBlock:
}
+class _FakeParameter:
+ def __init__(self, key: str) -> None:
+ self.key = key
+
+
class _FakeDefinition:
- def __init__(self, blocks: list[_FakeBlock]) -> None:
+ def __init__(self, blocks: list[_FakeBlock], parameters: list[_FakeParameter] | None = None) -> None:
self.blocks = blocks
+ self.parameters = parameters or []
class _FakeWorkflow:
@@ -213,6 +222,490 @@ def test_referenced_output_labels_finds_jinja_refs() -> None:
assert "a" in refs
+def test_referenced_output_labels_finds_block_form_jinja_refs() -> None:
+ new = _FakeDefinition(
+ [
+ _FakeBlock("extract_article_info", "extraction"),
+ _FakeBlock(
+ "summarize_article",
+ "text_prompt",
+ {
+ "prompt": (
+ "Summarize {{ extract_article_info.output.extracted_information.abstract }} "
+ "and {{ extract_article_info.title }}."
+ )
+ },
+ ),
+ ]
+ )
+
+ refs = _referenced_output_labels(["summarize_article"], new)
+
+ assert refs == {"extract_article_info"}
+
+
+def test_plan_frontier_append_with_block_form_jinja_ref_falls_back_to_full_run() -> None:
+ old = _FakeDefinition(
+ [
+ _FakeBlock("open_page", "navigation"),
+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),
+ ]
+ )
+ new = _FakeDefinition(
+ [
+ _FakeBlock("open_page", "navigation"),
+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),
+ _FakeBlock(
+ "summarize_article",
+ "text_prompt",
+ {
+ "prompt": (
+ "Summarize the main findings from "
+ "{{ extract_article_info.output.extracted_information.abstract }}."
+ )
+ },
+ ),
+ ]
+ )
+ ctx = _make_ctx()
+ ctx.verified_prefix_labels = ["open_page", "extract_article_info"]
+ ctx.verified_block_outputs = {
+ "open_page": "nav_ok",
+ "extract_article_info": {"extracted_information": {"abstract": "Prior output"}},
+ }
+
+ labels, seed, frontier = _plan_frontier(
+ ctx,
+ ["open_page", "extract_article_info", "summarize_article"],
+ old,
+ new,
+ )
+
+ assert labels == ["open_page", "extract_article_info", "summarize_article"]
+ assert seed == {}
+ assert frontier == "open_page"
+
+
+def test_plan_frontier_append_seeds_output_parameter_jinja_ref() -> None:
+ old = _FakeDefinition(
+ [
+ _FakeBlock("open_page", "navigation"),
+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),
+ ]
+ )
+ new = _FakeDefinition(
+ [
+ _FakeBlock("open_page", "navigation"),
+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),
+ _FakeBlock(
+ "summarize_article",
+ "text_prompt",
+ {
+ "prompt": (
+ "Summarize the main findings from "
+ "{{ extract_article_info_output.extracted_information.abstract }}."
+ )
+ },
+ ),
+ ]
+ )
+ ctx = _make_ctx()
+ ctx.verified_prefix_labels = ["open_page", "extract_article_info"]
+ ctx.verified_block_outputs = {
+ "open_page": "nav_ok",
+ "extract_article_info": {"extracted_information": {"abstract": "Prior output"}},
+ }
+
+ labels, seed, frontier = _plan_frontier(
+ ctx,
+ ["open_page", "extract_article_info", "summarize_article"],
+ old,
+ new,
+ )
+
+ assert labels == ["summarize_article"]
+ assert seed == {"extract_article_info": {"extracted_information": {"abstract": "Prior output"}}}
+ assert frontier == "summarize_article"
+
+
+def test_stale_metadata_detects_corrected_subject_label_and_title() -> None:
+ prior_yaml = """
+title: Count example.com topic alpha results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: extract_results
+ navigation_goal: Search example.com for topic alpha.
+ - block_type: extraction
+ label: extract_results
+ title: Extract Results
+ next_block_label: null
+ data_extraction_goal: Extract the total number of topic alpha search results.
+"""
+ submitted_yaml = """
+title: Count example.com sample beta results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: extract_results
+ navigation_goal: Search example.com for sample beta.
+ - block_type: extraction
+ label: extract_results
+ title: Extract Results
+ next_block_label: null
+ data_extraction_goal: Extract the total number of sample beta search results.
+"""
+
+ stale = tools._detect_stale_block_metadata(submitted_yaml, prior_yaml)
+
+ assert stale == [
+ {
+ "label": "search_topic_alpha",
+ "reasons": [
+ "label 'search_topic_alpha' appears stale",
+ "title 'Search Topic Alpha' appears stale",
+ ],
+ }
+ ]
+
+
+def test_stale_metadata_accepts_renamed_corrected_subject() -> None:
+ prior_yaml = """
+title: Count example.com topic alpha results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: extract_results
+ navigation_goal: Search example.com for topic alpha.
+ - block_type: extraction
+ label: extract_results
+ title: Extract Results
+ next_block_label: null
+ data_extraction_goal: Extract the total number of topic alpha search results.
+"""
+ submitted_yaml = """
+title: Count example.com sample beta results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_sample_beta
+ title: Search Sample Beta
+ next_block_label: extract_results
+ navigation_goal: Search example.com for sample beta.
+ - block_type: extraction
+ label: extract_results
+ title: Extract Results
+ next_block_label: null
+ data_extraction_goal: Extract the total number of sample beta search results.
+"""
+
+ assert tools._detect_stale_block_metadata(submitted_yaml, prior_yaml) == []
+
+
+def test_stale_metadata_accepts_reworded_action_with_same_subject() -> None:
+ prior_yaml = """
+title: Count example.com topic alpha results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for topic alpha.
+"""
+ submitted_yaml = """
+title: Count example.com topic alpha results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Find example.com pages about topic alpha.
+"""
+
+ assert tools._detect_stale_block_metadata(submitted_yaml, prior_yaml) == []
+
+
+def test_plan_frontier_unknown_jinja_root_falls_back_to_full_requested_list() -> None:
+ old = _FakeDefinition([_FakeBlock("open_page", "navigation")])
+ new = _FakeDefinition(
+ [
+ _FakeBlock("open_page", "navigation"),
+ _FakeBlock("summarize_article", "text_prompt", {"prompt": "Summarize {{ missing_block.abstract }}."}),
+ ]
+ )
+ ctx = _make_ctx()
+ ctx.verified_prefix_labels = ["open_page"]
+ ctx.verified_block_outputs = {"open_page": "nav_ok"}
+
+ labels, seed, frontier = _plan_frontier(ctx, ["open_page", "summarize_article"], old, new)
+
+ assert labels == ["open_page", "summarize_article"]
+ assert seed == {}
+ assert frontier == "open_page"
+
+
+def test_plan_frontier_falls_back_when_unknown_root_coexists_with_seedable_ref() -> None:
+ # Even when the suffix references a verified upstream output (so seeding
+ # would otherwise let us skip the prefix), an additional unknown Jinja
+ # root must still trigger the conservative full-rerun fallback.
+ old = _FakeDefinition(
+ [
+ _FakeBlock("open_page", "navigation"),
+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),
+ ]
+ )
+ new = _FakeDefinition(
+ [
+ _FakeBlock("open_page", "navigation"),
+ _FakeBlock("extract_article_info", "extraction", {"prompt": "extract abstract"}),
+ _FakeBlock(
+ "summarize_article",
+ "text_prompt",
+ {
+ "prompt": (
+ "Summarize {{ extract_article_info_output.extracted_information.abstract }} "
+ "with context {{ missing_block.note }}."
+ )
+ },
+ ),
+ ]
+ )
+ ctx = _make_ctx()
+ ctx.verified_prefix_labels = ["open_page", "extract_article_info"]
+ ctx.verified_block_outputs = {
+ "open_page": "nav_ok",
+ "extract_article_info": {"extracted_information": {"abstract": "Prior output"}},
+ }
+
+ labels, seed, frontier = _plan_frontier(
+ ctx,
+ ["open_page", "extract_article_info", "summarize_article"],
+ old,
+ new,
+ )
+
+ assert labels == ["open_page", "extract_article_info", "summarize_article"]
+ assert seed == {}
+ assert frontier == "open_page"
+
+
+def test_unknown_jinja_roots_ignores_credential_real_value_synthetic_roots() -> None:
+ new = _FakeDefinition(
+ [
+ _FakeBlock(
+ "login",
+ "login",
+ {"prompt": "Sign in with {{ creds_real_username }} / {{ creds_real_password }}."},
+ ),
+ ],
+ parameters=[_FakeParameter("creds")],
+ )
+
+ assert tools._unknown_jinja_roots(["login"], new) == set()
+
+
+def test_unknown_jinja_roots_ignores_conditional_branch_context_roots() -> None:
+ new = _FakeDefinition(
+ [
+ _FakeBlock(
+ "branch",
+ "conditional",
+ {
+ "expression": (
+ "{{ params.foo }} {{ outputs.bar }} {{ environment.region }} {{ env.flag }} {{ llm.model }}"
+ )
+ },
+ ),
+ ]
+ )
+
+ assert tools._unknown_jinja_roots(["branch"], new) == set()
+
+
+def test_stale_metadata_accepts_single_token_subject_change_as_known_limit() -> None:
+ prior_yaml = """
+title: Search results page
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_cats
+ title: Search Cats
+ next_block_label: null
+ navigation_goal: Search the directory for cats.
+"""
+ submitted_yaml = """
+title: Search results page
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_cats
+ title: Search Cats
+ next_block_label: null
+ navigation_goal: Search the directory for dogs.
+"""
+
+ # The code gate is a conservative backstop: it requires at least two
+ # removed metadata tokens before rejecting. Single-token subject swaps are
+ # expected to be handled by the prompt instruction to rename changed
+ # subject metadata.
+ assert tools._detect_stale_block_metadata(submitted_yaml, prior_yaml) == []
+
+
+def test_stale_metadata_detects_stale_title_after_label_rename() -> None:
+ prior_yaml = """
+title: Count example.com topic alpha results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for topic alpha.
+"""
+ submitted_yaml = """
+title: Count example.com sample beta results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_sample_beta
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for sample beta.
+"""
+
+ stale = tools._detect_stale_block_metadata(submitted_yaml, prior_yaml)
+
+ assert stale == [
+ {
+ "label": "search_sample_beta",
+ "reasons": ["title 'Search Topic Alpha' appears stale"],
+ }
+ ]
+
+
+def test_stale_metadata_detects_stale_block_inside_loop_blocks() -> None:
+ prior_yaml = """
+title: For-each search results
+workflow_definition:
+ blocks:
+ - block_type: for_loop
+ label: per_topic
+ loop_blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for topic alpha.
+"""
+ submitted_yaml = """
+title: For-each search results
+workflow_definition:
+ blocks:
+ - block_type: for_loop
+ label: per_topic
+ loop_blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for sample beta.
+"""
+
+ stale = tools._detect_stale_block_metadata(submitted_yaml, prior_yaml)
+
+ assert {item["label"] for item in stale} == {"search_topic_alpha"}
+
+
+def test_stale_metadata_message_indicates_truncation_when_over_limit() -> None:
+ items = [{"label": f"label_{i}", "reasons": [f"reason {i}"]} for i in range(7)]
+ message = tools._stale_block_metadata_message(items)
+ assert "and 2 more" in message
+
+
+def test_stale_metadata_message_omits_truncation_indicator_under_limit() -> None:
+ items = [{"label": f"label_{i}", "reasons": [f"reason {i}"]} for i in range(3)]
+ message = tools._stale_block_metadata_message(items)
+ assert "more" not in message
+
+
+def test_referenced_output_labels_ignores_non_block_jinja_roots() -> None:
+ new = _FakeDefinition(
+ [
+ _FakeBlock(
+ "summarize_article",
+ "text_prompt",
+ {"prompt": "Summarize {{ search_term.field }} for {{ loop.index }}."},
+ ),
+ ]
+ )
+
+ refs = _referenced_output_labels(["summarize_article"], new)
+
+ assert refs == set()
+
+
+def test_plan_frontier_append_only_with_workflow_param_does_not_fall_back() -> None:
+ old = _FakeDefinition(
+ [_FakeBlock("open_page", "navigation")],
+ parameters=[_FakeParameter("search_term")],
+ )
+ new = _FakeDefinition(
+ [
+ _FakeBlock("open_page", "navigation"),
+ _FakeBlock("search", "navigation", {"prompt": "Search for {{ search_term }} on this site"}),
+ ],
+ parameters=[_FakeParameter("search_term")],
+ )
+ ctx = _make_ctx()
+ ctx.verified_prefix_labels = ["open_page"]
+ ctx.verified_block_outputs = {"open_page": "nav_ok"}
+
+ labels, seed, frontier = _plan_frontier(ctx, ["open_page", "search"], old, new)
+
+ assert labels == ["search"]
+ assert seed == {}
+ assert frontier == "search"
+
+
+def test_template_builtin_roots_track_jinja_and_skyvern_contexts() -> None:
+ assert tools._JINJA_RUNTIME_GLOBAL_ROOTS == frozenset(SandboxedEnvironment().globals)
+ assert tools._JINJA_RUNTIME_GLOBAL_ROOTS <= tools._TEMPLATE_BUILTIN_ROOTS
+ assert tools._JINJA_LITERAL_ROOTS <= tools._TEMPLATE_BUILTIN_ROOTS
+ assert tools._JINJA_SPECIAL_CONTEXT_ROOTS <= tools._TEMPLATE_BUILTIN_ROOTS
+ assert frozenset(RESERVED_PARAMETER_KEYS) <= tools._SKYVERN_TEMPLATE_CONTEXT_ROOTS
+ assert {"parameters", "browser_session_id", "organization_id"} <= tools._SKYVERN_TEMPLATE_CONTEXT_ROOTS
+ assert tools._SKYVERN_TEMPLATE_CONTEXT_ROOTS <= tools._TEMPLATE_BUILTIN_ROOTS
+
+
+def test_unknown_jinja_roots_ignores_jinja_and_skyvern_context_roots() -> None:
+ new = _FakeDefinition(
+ [
+ _FakeBlock(
+ "summarize",
+ "text_prompt",
+ {
+ "prompt": (
+ "{{ range }} {{ dict }} {{ namespace }} {{ cycler }} {{ joiner }} {{ lipsum }} "
+ "{{ none }} {{ true }} {{ false }} {{ loop.index }} {{ self }} {{ varargs }} {{ kwargs }} "
+ "{{ parameters.search_term }} {{ browser_session_id }} {{ organization_id }} "
+ "{{ current_date }} {{ workflow_run_id }}"
+ )
+ },
+ ),
+ ]
+ )
+
+ assert tools._unknown_jinja_roots(["summarize"], new) == set()
+
+
# --------------------------------------------------------------------------- #
# Compact packet shape #
# --------------------------------------------------------------------------- #
@@ -402,6 +895,36 @@ def test_failed_unchanged_rerun_preserves_verified_prefix_and_outputs() -> None:
assert ctx.verified_block_outputs == {"a": "nav", "b": {"title": "hi"}}
+def test_run_blocks_outcome_rolls_forward_after_failed_preview() -> None:
+ ctx = _make_ctx()
+
+ tools._record_run_blocks_result(
+ ctx,
+ {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_fail",
+ "blocks": [{"label": "summarize", "status": "failed", "failure_reason": "Jinja ref undefined"}],
+ },
+ },
+ )
+ assert ctx.last_test_ok is False
+
+ tools._record_run_blocks_result(
+ ctx,
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_success",
+ "blocks": [{"label": "summarize", "status": "completed", "extracted_data": {"summary": "ok"}}],
+ },
+ },
+ )
+
+ assert ctx.last_test_ok is True
+ assert ctx.last_test_failure_reason is None
+
+
def test_yaml_diff_invalidation_drops_edited_label_and_downstream() -> None:
"""When the YAML changes between runs, verified-state invalidation based
on the diff should drop edited labels and anything downstream so the
diff --git a/tests/unit/test_copilot_session_injection.py b/tests/unit/test_copilot_session_injection.py
index fe75c1daa..ed1ef0029 100644
--- a/tests/unit/test_copilot_session_injection.py
+++ b/tests/unit/test_copilot_session_injection.py
@@ -459,6 +459,86 @@ class TestUpdateWorkflowDirect:
assert ctx.workflow_yaml == "original yaml"
+ @pytest.mark.asyncio
+ async def test_rejects_stale_block_metadata_before_persistence(self) -> None:
+ from skyvern.forge.sdk.copilot.tools import _update_workflow
+
+ prior_yaml = """
+title: Count example.com topic alpha results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for topic alpha.
+"""
+ submitted_yaml = """
+title: Count example.com sample beta results
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: search_topic_alpha
+ title: Search Topic Alpha
+ next_block_label: null
+ navigation_goal: Search example.com for sample beta.
+"""
+ ctx = _make_ctx(workflow_yaml=prior_yaml)
+
+ result = await _update_workflow({"workflow_yaml": submitted_yaml}, ctx)
+
+ assert result["ok"] is False
+ assert "corrected block metadata still appears stale" in result["error"]
+ assert "search_topic_alpha" in result["error"]
+ assert ctx.workflow_yaml == prior_yaml
+
+ @pytest.mark.asyncio
+ async def test_stale_check_prefers_in_turn_emission_over_turn_start_yaml(self) -> None:
+ # Cross-path flow: an earlier inline REPLACE_WORKFLOW left
+ # ctx.last_workflow_yaml at the renamed state. A subsequent
+ # update_workflow call submits a regression that re-stales the label
+ # — this must compare against ctx.last_workflow_yaml, not the older
+ # ctx.workflow_yaml that still points at turn-start.
+ from skyvern.forge.sdk.copilot.tools import _update_workflow
+
+ turn_start_yaml = """
+title: Alpha workflow
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: review_alpha_one
+ title: Review Alpha One
+ next_block_label: null
+ navigation_goal: Review the alpha one records on the directory.
+"""
+ in_turn_emission_yaml = """
+title: Beta workflow
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: review_beta_two
+ title: Review Beta Two
+ next_block_label: null
+ navigation_goal: Review the beta two records on the directory.
+"""
+ regressed_yaml = """
+title: Beta workflow
+workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: review_beta_two
+ title: Review Beta Two
+ next_block_label: null
+ navigation_goal: Review the gamma three records on the directory.
+"""
+ ctx = _make_ctx(workflow_yaml=turn_start_yaml)
+ ctx.last_workflow_yaml = in_turn_emission_yaml
+
+ result = await _update_workflow({"workflow_yaml": regressed_yaml}, ctx)
+
+ assert result["ok"] is False
+ assert "review_beta_two" in result["error"]
+
class TestWorkflowUpdatePersistence:
def test_record_marks_persisted_even_when_workflow_has_zero_blocks(self) -> None:
diff --git a/tests/unit/test_workflow_copilot_prompt_injection.py b/tests/unit/test_workflow_copilot_prompt_injection.py
index f9a75524c..46dd8a0f0 100644
--- a/tests/unit/test_workflow_copilot_prompt_injection.py
+++ b/tests/unit/test_workflow_copilot_prompt_injection.py
@@ -64,6 +64,15 @@ class TestSystemTemplateSecurity:
assert "TRUSTED_KB_CONTENT" in rendered
+class TestAgentTemplateCorrectionRules:
+ def test_agent_template_requires_label_and_title_refresh_on_corrections(self) -> None:
+ rendered = prompt_engine.load_prompt("workflow-copilot-agent", **_AGENT_TEMPLATE_DEFAULTS)
+
+ assert "Rename affected block labels and block titles" in rendered
+ assert "Labels become output keys" in rendered
+ assert "Jinja block reference" in rendered
+
+
class TestUserTemplateCodeFencing:
"""Verify untrusted variables are wrapped in code fences."""