Skip to content

Automate browser based workflows with AI

perf(SKY-12171): eliminate measured slow-test hotspots (#7262)

Aaron Perez160f28dc7fc5
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index b7462c12a..a46f8b675 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -1,5 +1,5 @@
Classify the latest Workflow Copilot request. Return only JSON with keys:
-testing_intent=require_test|skip_test|unspecified; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
+testing_intent=require_test|skip_test|unspecified; authoring_intent=author_now|defer_authoring; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
testing_intent:
- skip_test: the user asks for an untested draft, says they will test it themselves, says draft only, just sketch it, no validation run, or skip testing.
@@ -7,6 +7,11 @@ testing_intent:
- unspecified: the user does not say whether Skyvern should test the workflow.
- Do not use skip_test for a "done when", "complete when", or success-condition sentence; that is a completion_contract.
+authoring_intent:
+- defer_authoring: only when the latest message BOTH asks to act on the live page/form now (fill, enter, click, submit it on the open page) AND explicitly says not to author, build, save, or create a workflow yet. Both halves must be present.
+- author_now: the default for every other request, including an ordinary "build a workflow that fills the form" — building a workflow IS the author-now case.
+- Do not infer defer_authoring from a live-action verb alone, from skip_test, or from a draft-only request; author-deferral requires the explicit "do not author the workflow yet" half.
+
credential_input_kind:
- raw_secret: the user pasted a literal credential value — a password string, an API key/bearer token value, a session/auth cookie value, a TOTP/OTP/MFA digit code, a private-key body, a card number, or a CVV. The value must look like the secret itself (e.g. `password: hunter2`, `[email protected]:<password>`, bulk `[email protected]:<password>` account rows, `eyJ…` JWT, `sk-…` API key, a numeric OTP). Identifier-shaped vault-pointer strings such as `customer-<uuid>-pass`, `prod-app-secret-id`, `<service>-key-vault-id`, or `https://<vault>.vault.azure.net/secrets/<name>` are POINTERS to where a secret is stored, not secret values; do NOT classify them as raw_secret on the basis of a `-pass`, `-secret`, `-key`, or `-token` suffix alone.
- credential_id: the user provided one or more exact credential IDs beginning with `cred_`. Put every ID in credential_refs.
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index b4156db24..f5036ebe4 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -44,7 +44,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
terminal_evidence_has_recorded_state,
)
from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data
-from skyvern.forge.sdk.copilot.build_phase import initial_build_phase
+from skyvern.forge.sdk.copilot.build_phase import BuildPhase, initial_build_phase
from skyvern.forge.sdk.copilot.build_test_outcome import (
_VALUE_EXCERPT_MAX,
RecordedBuildTestOutcome,
@@ -152,6 +152,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
RequestPolicy,
build_request_policy,
credential_prompt_reason,
+ is_defer_authoring_durable_fill_criterion,
redact_raw_secrets_for_prompt,
)
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcome_display_reason
@@ -177,10 +178,12 @@ from skyvern.forge.sdk.copilot.turn_intent import (
RequiredContextKey,
TurnIntent,
TurnIntentClassifierResult,
+ TurnIntentExpectedOutput,
TurnIntentMode,
TurnIntentReasonCode,
build_turn_intent,
classify_turn_intent,
+ turn_intent_defers_authoring_live_fill,
)
from skyvern.forge.sdk.copilot.turn_outcome import (
apply_repeated_reply_guard,
@@ -459,12 +462,14 @@ def _reconcile_completion_criteria_on_context(
policy_inputs: RequestPolicyGuardrailInputs,
) -> None:
fresh_criteria = list(policy.completion_criteria)
+ durable_fill_carriers = [c for c in fresh_criteria if is_defer_authoring_durable_fill_criterion(c)]
floored_fresh, fresh_floor_rekeyed_paths = apply_requested_output_producer_floor(fresh_criteria)
if fresh_floor_rekeyed_paths:
policy.completion_criteria = list(floored_fresh)
snapshot = policy_inputs.stored_completion_criteria
if snapshot is None:
_log_requested_output_producer_floor(fresh_floor_rekeyed_paths)
+ _restore_durable_fill_carriers(policy, durable_fill_carriers)
return
requested_output_path_aliases = (
ctx.copilot_config.requested_output_path_aliases if ctx.copilot_config is not None else None
@@ -482,6 +487,16 @@ def _reconcile_completion_criteria_on_context(
if decision.action == "adopt_stored" or floor_rekeyed_paths:
policy.completion_criteria = list(floored_criteria)
_log_requested_output_producer_floor(floor_rekeyed_paths)
+ _restore_durable_fill_carriers(policy, durable_fill_carriers)
+
+
+def _restore_durable_fill_carriers(policy: RequestPolicy, carriers: list[CompletionCriterion]) -> None:
+ if not carriers:
+ return
+ present_ids = {criterion.id for criterion in policy.completion_criteria}
+ missing = [carrier for carrier in carriers if carrier.id not in present_ids]
+ if missing:
+ policy.completion_criteria = list(policy.completion_criteria) + missing
def _store_request_policy_on_context(
@@ -3599,6 +3614,15 @@ def _structural_infeasibility_question(turn_intent: TurnIntent | None) -> str |
return question or None
+def _infeasibility_bail_question(turn_intent: TurnIntent | None) -> str | None:
+ """The pre-loop clarification question, or None when the turn must not bail. A
+ defer-authoring live-fill turn decides feasibility post-session at the actuation
+ hook, so it never takes the pre-loop bail even when judged structurally infeasible."""
+ if turn_intent_defers_authoring_live_fill(turn_intent):
+ return None
+ return _structural_infeasibility_question(turn_intent)
+
+
def _build_infeasibility_clarification_result(
question: str,
user_message: str,
@@ -3649,6 +3673,86 @@ def _build_infeasibility_clarification_result(
)
+_DEFER_AUTHORING_NO_SESSION_MESSAGE = (
+ "You asked me to fill the page live now instead of authoring a workflow, but I don't have a live "
+ "browser session open on that page to act on. Open the page in a live browser session and I'll fill it directly."
+)
+
+
+def apply_defer_authoring_actuation_entry(
+ ctx: CopilotContext,
+ validated_browser_session_id: str | None,
+ *,
+ user_message: str,
+ prior_global_llm_context: str | None,
+ prior_workflow_yaml: str | None,
+) -> AgentResult | None:
+ """With a validated live session, enters COMPOSING and enables live actuation while workflow
+ mutation stays blocked by the authority gate; without one, returns a deterministic honest terminal."""
+ turn_intent = ctx.turn_intent
+ if not isinstance(turn_intent, TurnIntent):
+ return None
+ if not turn_intent_defers_authoring_live_fill(turn_intent):
+ return None
+ if not validated_browser_session_id:
+ return _build_defer_authoring_no_session_result(
+ user_message=user_message,
+ prior_global_llm_context=prior_global_llm_context,
+ prior_workflow_yaml=prior_workflow_yaml,
+ ctx=ctx,
+ )
+ if turn_intent.mode in NO_MUTATION_TURN_INTENT_MODES:
+ turn_intent.mode = TurnIntentMode.BUILD
+ turn_intent.expected_output = TurnIntentExpectedOutput.RUN_RESULT
+ if RequiredContextKey.BROWSER_STATE not in turn_intent.required_context:
+ turn_intent.required_context.append(RequiredContextKey.BROWSER_STATE)
+ ctx.build_phase = BuildPhase.COMPOSING
+ return None
+
+
+def _build_defer_authoring_no_session_result(
+ *,
+ user_message: str,
+ prior_global_llm_context: str | None,
+ prior_workflow_yaml: str | None,
+ ctx: CopilotContext,
+) -> AgentResult:
+ structured = StructuredContext.from_json_str(prior_global_llm_context)
+ if not structured.user_goal:
+ structured.user_goal = user_message[:300]
+ structured.decisions_made.append("defer-authoring live fill deferred: no live browser session")
+
+ final_text, outcome = apply_repeated_reply_guard(
+ final_text=_DEFER_AUTHORING_NO_SESSION_MESSAGE,
+ attempted_kind=ResponseKind.CLARIFY,
+ blocked_signatures=list(ctx.blocked_reply_signatures),
+ reason_code="defer_authoring_no_live_session",
+ )
+ return _finalize_result_with_blocker_override(
+ ctx,
+ _make_agent_result(
+ ctx,
+ user_response=final_text,
+ updated_workflow=None,
+ global_llm_context=structured.to_json_str(),
+ response_type="ASK_QUESTION",
+ workflow_yaml=prior_workflow_yaml or None,
+ workflow_was_persisted=False,
+ clear_proposed_workflow=not outcome_fully_verified(ctx),
+ turn_outcome=outcome,
+ turn_id=ctx.turn_id,
+ narrative_summary=ctx.narrative_summary,
+ narrative_payload=_build_narrative_payload(
+ ctx,
+ terminal="response",
+ terminal_message=final_text,
+ narrative_summary=ctx.narrative_summary,
+ ),
+ ),
+ exit_site="defer_authoring_no_live_session",
+ )
+
+
def _fallback_llm_key(config: CopilotConfig, current_llm_key: str) -> str | None:
fallback_key = config.fallback_llm_key
if not fallback_key or fallback_key == current_llm_key:
@@ -4476,7 +4580,7 @@ async def _run_copilot_turn_impl(
)
# Infeasibility rides on turn_intent: a verdict carrying a question bails to a pre-loop clarification.
- infeasibility_question = _structural_infeasibility_question(ctx.turn_intent)
+ infeasibility_question = _infeasibility_bail_question(ctx.turn_intent)
if infeasibility_question is not None:
return _build_infeasibility_clarification_result(
question=infeasibility_question,
@@ -4508,6 +4612,15 @@ async def _run_copilot_turn_impl(
validated_browser_session_id = await _resolve_live_browser_session_id(chat_request, organization_id)
ctx.browser_session_id = validated_browser_session_id
+ defer_authoring_terminal = apply_defer_authoring_actuation_entry(
+ ctx,
+ validated_browser_session_id,
+ user_message=agent_user_message,
+ prior_global_llm_context=global_llm_context,
+ prior_workflow_yaml=chat_request.workflow_yaml,
+ )
+ if defer_authoring_terminal is not None:
+ return defer_authoring_terminal
model_name, run_config, llm_key, supports_vision = resolve_model_config(
llm_api_handler,
diff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.py
index 641988fcd..36310aeb7 100644
--- a/skyvern/forge/sdk/copilot/code_block_preflight.py
+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py
@@ -40,6 +40,9 @@ class CodeBlockSandboxNameDiagnostic:
_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*m|\x1b\([AB]")
_MYPY_ERROR_RE = re.compile(r"^(?P<path>.*?):(?P<line>\d+): (?P<severity>error): (?P<message>.*)$")
_LOCATOR_NOT_CALLABLE_RE = re.compile(r'"Locator" not callable\s+\[operator\]')
+# Locator properties that return a Locator; calling one (``.first()`` / ``.last()``)
+# is the ``"Locator" not callable`` misuse the mypy pass exists to surface.
+_LOCATOR_NONCALLABLE_PROPERTIES = frozenset({"first", "last"})
_BROAD_BODY_TEXT_WAIT_NEEDLES = (
"document.body.innertext",
"document.body.textcontent",
@@ -232,6 +235,13 @@ def preflight_code_block(
if diagnostics:
return diagnostics
+ # Booting mypy costs ~1s per call, and the only diagnostic it can surface is
+ # ``"Locator" not callable`` (a Locator invoked as a function). Skip the boot
+ # entirely when a cheap AST scan proves the snippet contains no such call.
+ tree, _ = _parse_static_ast(code)
+ if tree is None or not _may_invoke_locator_object(tree):
+ return []
+
try:
from mypy import api as mypy_api
except ImportError:
@@ -264,6 +274,26 @@ def preflight_code_block(
return _parse_mypy_output(stdout)
+def _may_invoke_locator_object(tree: ast.AST) -> bool:
+ """True when the snippet could call a Locator object as a function.
+
+ ``"Locator" not callable`` is the only diagnostic the mypy pass surfaces, and
+ it arises from calling a Locator-returning property (``.first()`` / ``.last()``)
+ or the result of another call (``page.locator(...)()``). Conservative: any such
+ shape returns True so the mypy pass still runs; only snippets provably free of
+ them skip it.
+ """
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ func = node.func
+ if isinstance(func, ast.Attribute) and func.attr in _LOCATOR_NONCALLABLE_PROPERTIES:
+ return True
+ if isinstance(func, ast.Call):
+ return True
+ return False
+
+
def sandbox_unresolved_name_diagnostics(
code: str,
*,
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 840c9c890..5f975f6c1 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -32,6 +32,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
_coerce_requested_output_evidence_source,
_normalize_contingent_antecedent_output_path,
_normalize_deliverable_kind,
+ is_defer_authoring_durable_fill_criterion,
is_fallback_floor_criterion,
judgment_truth_condition_key,
normalized_criterion_outcome_key,
@@ -426,6 +427,7 @@ def reconcile_completion_criteria(
"""
stored = snapshot.active if snapshot is not None else None
next_epoch = snapshot.next_epoch if snapshot is not None else 1
+ fresh = [criterion for criterion in fresh if not is_defer_authoring_durable_fill_criterion(criterion)]
if fresh and all(is_fallback_floor_criterion(criterion) for criterion in fresh):
fresh = []
if stored is None:
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 3376e3e31..1f306cb6e 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -180,6 +180,10 @@ MAX_NO_PROGRESS_INTERACTION_ATTEMPTS = 4
_NO_PROGRESS_INTERACTION_REASON_CODES = frozenset({"loop_detected_no_forward_progress_interaction"})
MIN_BLOCKS_FOR_AUTO_COMPLETE = 10
TOTAL_TIMEOUT_SECONDS = 900
+# Floor for the per-iteration ``wait_for`` deadline so an already-spent budget
+# never yields ``wait_for(timeout=0)`` (which raises immediately). Kept as a
+# constant so tests can shrink it instead of paying a full second per deadline.
+MIN_DEADLINE_REMAINING_SECONDS = 1.0
# Belt-and-braces cap alongside the elapsed-time budget. Per-nudge caps
# already prevent individual branches from looping; this stops a brand-new
# enforcement rule that forgets its own counter from spinning within 900s.
@@ -2024,13 +2028,13 @@ async def _run_streamed_with_deadline(
``_build_exit_result`` path emits a non-empty REPLY before the
client's own transport timeout closes the stream.
- ``max(1.0, ...)`` floors ``remaining`` so ``wait_for(timeout=0)``
- never panics on an already-spent budget.
+ ``MIN_DEADLINE_REMAINING_SECONDS`` floors ``remaining`` so
+ ``wait_for(timeout=0)`` never panics on an already-spent budget.
"""
from skyvern.forge.sdk.copilot.streaming_adapter import stream_to_sse
elapsed = time.monotonic() - start_time
- remaining = max(1.0, TOTAL_TIMEOUT_SECONDS - elapsed)
+ remaining = max(MIN_DEADLINE_REMAINING_SECONDS, TOTAL_TIMEOUT_SECONDS - elapsed)
result = Runner.run_streamed(agent, input=current_input, context=ctx, session=session, **runner_kwargs)
try:
try:
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 4b93ae38e..6cd2ebf40 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -36,6 +36,8 @@ from skyvern.utils.yaml_loader import safe_load_no_dates
LOG = structlog.get_logger()
PROMPT_NAME = "workflow-copilot-request-policy"
_TESTING_INTENTS = {"require_test", "skip_test", "unspecified"}
+_AUTHORING_INTENTS = {"author_now", "defer_authoring"}
+DEFER_AUTHORING_DURABLE_FILL_CRITERION_ID = "defer_authoring_durable_fill"
_KINDS = {"none", "raw_secret", "credential_id", "credential_name", "website_stored_credential", "placeholder"}
_RAW_SECRET_HANDLINGS = {"none", "block", "redacted_draft"}
_CLASSIFIER_FAILURE_KINDS = {
@@ -48,6 +50,7 @@ _CLASSIFIER_FAILURE_KINDS = {
}
_CLASSIFICATION_RESPONSE_FIELDS = {
"testing_intent",
+ "authoring_intent",
"credential_input_kind",
"credential_refs",
"login_page_urls",
@@ -297,6 +300,7 @@ class CompletionCriterion:
@dataclass
class RequestPolicy:
testing_intent: str = "unspecified"
+ authoring_intent: str = "author_now"
credential_input_kind: str = "none"
credential_refs: list[str] = field(default_factory=list)
login_page_urls: list[str] = field(default_factory=list)
@@ -336,6 +340,7 @@ class RequestPolicy:
def to_trace_data(self) -> dict[str, Any]:
data: dict[str, Any] = {
"testing_intent": self.testing_intent,
+ "authoring_intent": self.authoring_intent,
"credential_input_kind": self.credential_input_kind,
"clarification_reason": self.clarification_reason,
"allow_update_workflow": self.allow_update_workflow,
@@ -389,6 +394,7 @@ class RequestPolicy:
def prompt_summary(self) -> str:
lines = [
f"testing_intent: {self.testing_intent}",
+ f"authoring_intent: {self.authoring_intent}",
f"credential_input_kind: {self.credential_input_kind}",
f"clarification_reason: {self.clarification_reason}",
f"allow_update_workflow: {self.allow_update_workflow}",
@@ -434,6 +440,21 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicy
return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)
+def is_defer_authoring_durable_fill_criterion(criterion: CompletionCriterion) -> bool:
+ return criterion.id == DEFER_AUTHORING_DURABLE_FILL_CRITERION_ID
+
+
+def _defer_authoring_durable_fill_criterion() -> CompletionCriterion:
+ return CompletionCriterion(
+ id=DEFER_AUTHORING_DURABLE_FILL_CRITERION_ID,
+ outcome="the live form is filled on the page this turn",
+ kind="terminal_action",
+ terminal_action_family="form",
+ method_mandated=True,
+ level="run",
+ )
+
+
def credential_prompt_reason(policy: RequestPolicy | None, final_text: str | None) -> str | None:
# Typed clarification_reason wins, then the explicit-defer flag — narrowly, since
# allow_missing_credentials_in_draft alone also covers the generic skip_test
@@ -751,6 +772,7 @@ def _classification_from_raw(raw: Any) -> RequestPolicy:
if raw is None:
return RequestPolicy()
testing_intent = raw.get("testing_intent")
+ authoring_intent = raw.get("authoring_intent")
credential_input_kind = raw.get("credential_input_kind")
completion_contract_raw = raw.get("completion_contract")
completion_contract = completion_contract_raw.strip() if isinstance(completion_contract_raw, str) else None
@@ -758,6 +780,7 @@ def _classification_from_raw(raw: Any) -> RequestPolicy:
raw_secret_evidence = evidence_raw if isinstance(evidence_raw, str) and evidence_raw.strip() else None
policy = RequestPolicy(
testing_intent=testing_intent if testing_intent in _TESTING_INTENTS else "unspecified",
+ authoring_intent=authoring_intent if authoring_intent in _AUTHORING_INTENTS else "author_now",
credential_input_kind=credential_input_kind if credential_input_kind in _KINDS else "none",
credential_refs=_clean_list(raw.get("credential_refs") or []),
login_page_urls=_clean_list(raw.get("login_page_urls") or []),
@@ -2695,6 +2718,12 @@ async def build_request_policy(
"I could not verify the requested credential metadata for this organization. Please provide a valid saved credential by exact name or a credential ID beginning with cred_.",
)
+ if policy.authoring_intent == "defer_authoring":
+ policy.allow_update_workflow = False
+ policy.allow_run_blocks = False
+ if not any(is_defer_authoring_durable_fill_criterion(criterion) for criterion in policy.completion_criteria):
+ policy.completion_criteria = list(policy.completion_criteria) + [_defer_authoring_durable_fill_criterion()]
+
trace_data = policy.to_trace_data()
if policy.classifier_status == "fallback":
LOG.warning("request-policy fallback policy used", **trace_data)
diff --git a/skyvern/forge/sdk/copilot/turn_intent.py b/skyvern/forge/sdk/copilot/turn_intent.py
index 7cc3f3e01..886ead287 100644
--- a/skyvern/forge/sdk/copilot/turn_intent.py
+++ b/skyvern/forge/sdk/copilot/turn_intent.py
@@ -100,6 +100,7 @@ class TurnIntentReasonCode(StrEnum):
DEFAULT_UNKNOWN = "default_unknown"
REQUEST_POLICY_DERIVED = "request_policy_derived"
REQUEST_POLICY_CLARIFICATION = "request_policy_clarification"
+ AUTHORING_INTENT_DEFER_AUTHORING = "authoring_intent_defer_authoring"
TESTING_INTENT_SKIP_TEST = "testing_intent_skip_test"
TESTING_INTENT_RUN_OVERRIDES_DIAGNOSE = "testing_intent_run_overrides_diagnose"
WORKFLOW_CONTEXT_PRESENT = "workflow_context_present"
@@ -128,8 +129,14 @@ class TurnIntentClassifierFailureKind(StrEnum):
MALFORMED_OUTPUT = "malformed_output"
+_DETERMINISTIC_ONLY_REASON_CODES = frozenset(
+ {
+ TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK,
+ TurnIntentReasonCode.AUTHORING_INTENT_DEFER_AUTHORING,
+ }
+)
_CLASSIFIER_REASON_CODES = tuple(
- reason for reason in TurnIntentReasonCode if reason != TurnIntentReasonCode.TRANSIENT_CLASSIFIER_FALLBACK
+ reason for reason in TurnIntentReasonCode if reason not in _DETERMINISTIC_ONLY_REASON_CODES
)
@@ -785,6 +792,8 @@ def build_turn_intent(
reason_codes.append(TurnIntentReasonCode.BROWSER_CONTEXT_PRESENT)
if request_policy.credential_input_kind != "none" or request_policy.resolved_credentials:
required_context.append(RequiredContextKey.CREDENTIAL_METADATA)
+ if request_policy.authoring_intent == "defer_authoring" and not request_policy.raw_secret_detected:
+ reason_codes.append(TurnIntentReasonCode.AUTHORING_INTENT_DEFER_AUTHORING)
authority = TurnIntentAuthority(
may_update_workflow=request_policy.allow_update_workflow,
@@ -962,3 +971,10 @@ def build_turn_intent(
reason_codes=reason_codes,
missing_context_question=missing_context_question,
)
+
+
+def turn_intent_defers_authoring_live_fill(turn_intent: TurnIntent | None) -> bool:
+ return (
+ isinstance(turn_intent, TurnIntent)
+ and TurnIntentReasonCode.AUTHORING_INTENT_DEFER_AUTHORING in turn_intent.reason_codes
+ )
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 72fc572b7..432f82139 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -54,6 +54,7 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (
format_recoverable_failure_reply,
merge_failure_into_context,
)
+from skyvern.forge.sdk.copilot.request_policy import is_defer_authoring_durable_fill_criterion
from skyvern.forge.sdk.copilot.turn_outcome import (
CopilotComposerMode,
build_minimal_turn_outcome,
@@ -600,7 +601,9 @@ async def _persist_completion_criteria_state(chat: Any, agent_result: AgentResul
organization_id=chat.organization_id,
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
goal_epoch=plan.create_epoch or 1,
- criteria=criteria_to_json(plan.create_criteria),
+ criteria=criteria_to_json(
+ [c for c in plan.create_criteria if not is_defer_authoring_durable_fill_criterion(c)]
+ ),
source_turn_id=agent_result.turn_id,
source_goal_text=user_message[:2000] if user_message else None,
consecutive_all_no_evidence=plan.counter_value,
diff --git a/skyvern/webeye/navigation.py b/skyvern/webeye/navigation.py
index 78e580a07..c98a9c77e 100644
--- a/skyvern/webeye/navigation.py
+++ b/skyvern/webeye/navigation.py
@@ -39,8 +39,11 @@ async def navigate_with_retry(
retry_times: int,
settle: SettleFunc,
wait_until: Literal["load", "domcontentloaded", "commit"] = "load",
- sleep: SleepFunc = asyncio.sleep,
+ sleep: SleepFunc | None = None,
) -> None:
+ # Late-bound so a test patching ``asyncio.sleep`` reaches the retry backoff.
+ if sleep is None:
+ sleep = asyncio.sleep
degradation = _DEGRADATION_MAP.get(wait_until, [wait_until])
for attempt in range(retry_times):
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index 004953b7a..ddcac3a90 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -1,19 +1,27 @@
"""Shared pytest fixtures and setup for unit tests."""
# -- begin speed up unit tests
+import itertools
import logging
+import shutil
+from collections.abc import AsyncGenerator, Callable
+from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
+import pytest_asyncio
import structlog
from opentelemetry import trace as otel_trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from sqlalchemy import create_engine
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.db.models import Base
from tests.unit.force_stub_app import start_forge_stub_app
# Wire structlog through stdlib so caplog can capture log records in tests.
@@ -135,3 +143,48 @@ def span_exporter() -> InMemorySpanExporter:
exporter.clear()
yield exporter
exporter.clear()
+
+
+# -- shared in-memory SQLite engine for repository/route unit tests --
+#
+# ``Base.metadata.create_all`` issues DDL for every mapped table (~50) on every
+# call, so re-running it per test dominates the runtime of the repository suites.
+# We build the schema once per session into a template SQLite file and clone that
+# file per test — a byte copy is orders of magnitude cheaper than re-emitting the
+# DDL, and each test still gets its own isolated database.
+
+
[email protected](scope="session")
+def sqlite_schema_template(tmp_path_factory: pytest.TempPathFactory) -> Path:
+ template_path = tmp_path_factory.mktemp("sqlite_schema") / "schema.db"
+ engine = create_engine(f"sqlite:///{template_path}")
+ try:
+ Base.metadata.create_all(engine)
+ finally:
+ engine.dispose()
+ return template_path
+
+
+@pytest_asyncio.fixture
+async def sqlite_engine_factory(
+ sqlite_schema_template: Path, tmp_path: Path
+) -> AsyncGenerator[Callable[[], AsyncEngine]]:
+ engines: list[AsyncEngine] = []
+ counter = itertools.count()
+
+ def _make() -> AsyncEngine:
+ db_path = tmp_path / f"db_{next(counter)}.db"
+ shutil.copyfile(sqlite_schema_template, db_path)
+ engine = create_async_engine(f"sqlite+aiosqlite:///{db_path}")
+ engines.append(engine)
+ return engine
+
+ yield _make
+
+ for engine in engines:
+ await engine.dispose()
+
+
+@pytest_asyncio.fixture
+async def sqlite_engine(sqlite_engine_factory: Callable[[], AsyncEngine]) -> AsyncEngine:
+ return sqlite_engine_factory()
diff --git a/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py
index daf6c40f2..25bd15e18 100644
--- a/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py
+++ b/tests/unit/forge/sdk/routes/streaming/test_exfiltration_channel.py
@@ -3,7 +3,6 @@
from __future__ import annotations
import asyncio
-import gc
import json
import typing as t
import weakref
@@ -99,7 +98,15 @@ class TestExfiltrationChannelEvents:
def test_page_tracking_releases_collected_pages(self) -> None:
channel, _ = _make_channel()
- page = _make_page()
+
+ # A real MagicMock page carries internal reference cycles, forcing a
+ # full-heap gc.collect() to reclaim it (seconds late in the suite). A
+ # cycle-free stand-in is reclaimed by refcounting the moment the last
+ # strong reference drops, so the weak trackers clear without a collect.
+ class _Page:
+ pass
+
+ page = _Page()
page_ref = weakref.ref(page)
ExfiltrationChannel._active_binding_channels[page] = channel
@@ -107,7 +114,6 @@ class TestExfiltrationChannelEvents:
channel._page_console_captures[page] = PageConsoleCapture(console_listener=MagicMock())
del page
- gc.collect()
assert page_ref() is None
assert not ExfiltrationChannel._active_binding_channels
diff --git a/tests/unit/test_animation_wait_attrs.py b/tests/unit/test_animation_wait_attrs.py
index f82b6e9e5..6fd48e76f 100644
--- a/tests/unit/test_animation_wait_attrs.py
+++ b/tests/unit/test_animation_wait_attrs.py
@@ -34,7 +34,15 @@ def test_all_production_call_sites_pass_caller() -> None:
fpath = os.path.join(dirpath, fname)
try:
with open(fpath) as f:
- tree = ast.parse(f.read(), filename=fpath)
+ content = f.read()
+ except OSError:
+ continue
+ # A call site must textually contain the method name; skip parsing
+ # the thousands of files that cannot possibly hold one.
+ if "safe_wait_for_animation_end" not in content:
+ continue
+ try:
+ tree = ast.parse(content, filename=fpath)
except SyntaxError:
continue
for node in ast.walk(tree):
diff --git a/tests/unit/test_cached_download_s3_upload.py b/tests/unit/test_cached_download_s3_upload.py
index f86a493e4..84e157cdd 100644
--- a/tests/unit/test_cached_download_s3_upload.py
+++ b/tests/unit/test_cached_download_s3_upload.py
@@ -234,7 +234,8 @@ async def test_verification_retries_before_failing(setup, tmp_path):
try:
from skyvern.services.script_service import download
- await download(prompt="Download invoice", label="test_block")
+ with patch(f"{MODULE}.asyncio.sleep", new_callable=AsyncMock):
+ await download(prompt="Download invoice", label="test_block")
refs["fallback"].assert_not_called()
assert refs["storage"].get_downloaded_files.call_count == 4 # 1 before + 3 retries
@@ -308,7 +309,8 @@ async def test_get_before_timeout_skips_verification(setup, tmp_path):
try:
from skyvern.services.script_service import download
- await download(prompt="Download invoice", label="test_block")
+ with patch(f"{MODULE}.asyncio.sleep", new_callable=AsyncMock):
+ await download(prompt="Download invoice", label="test_block")
# Should NOT fall back — local verification passed, S3 verification skipped
refs["fallback"].assert_not_called()
@@ -337,7 +339,8 @@ async def test_get_after_timeout_skips_verification(setup, tmp_path):
try:
from skyvern.services.script_service import download
- await download(prompt="Download invoice", label="test_block")
+ with patch(f"{MODULE}.asyncio.sleep", new_callable=AsyncMock):
+ await download(prompt="Download invoice", label="test_block")
refs["fallback"].assert_not_called()
finally:
diff --git a/tests/unit/test_copilot_runner_deadline.py b/tests/unit/test_copilot_runner_deadline.py
index 88d108767..480b58df6 100644
--- a/tests/unit/test_copilot_runner_deadline.py
+++ b/tests/unit/test_copilot_runner_deadline.py
@@ -28,6 +28,7 @@ async def test_runner_deadline_raises_total_timeout_when_tool_exceeds_budget(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.05)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.MIN_DEADLINE_REMAINING_SECONDS", 0.02)
stream = MagicMock()
stream.is_disconnected = AsyncMock(return_value=False)
@@ -62,6 +63,7 @@ async def test_runner_deadline_protects_context_overflow_recovery_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 0.05)
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.MIN_DEADLINE_REMAINING_SECONDS", 0.02)
stream = MagicMock()
stream.is_disconnected = AsyncMock(return_value=False)
diff --git a/tests/unit/test_copilot_turn_intent.py b/tests/unit/test_copilot_turn_intent.py
index 42f9e8da0..90372c048 100644
--- a/tests/unit/test_copilot_turn_intent.py
+++ b/tests/unit/test_copilot_turn_intent.py
@@ -32,6 +32,7 @@ from skyvern.forge.sdk.copilot.turn_intent import (
_turn_intent_classification_from_raw,
build_turn_intent,
classify_turn_intent,
+ turn_intent_defers_authoring_live_fill,
)
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
@@ -341,6 +342,68 @@ def test_build_turn_intent_routes_raw_secret_to_refuse_over_llm_classification()
assert TurnIntentReasonCode.LLM_CLASSIFIER not in intent.reason_codes
+def test_build_turn_intent_defer_authoring_leaves_refuse_uncoerced_but_marks_reason() -> None:
+ policy = RequestPolicy(
+ authoring_intent="defer_authoring",
+ allow_update_workflow=False,
+ allow_run_blocks=False,
+ )
+
+ intent = build_turn_intent(
+ user_message="Fill the form on the open page now. Do not author or save a workflow yet.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=policy,
+ classifier_result=_classification(TurnIntentMode.REFUSE),
+ )
+
+ assert intent.mode == TurnIntentMode.REFUSE
+ assert intent.authority.may_update_workflow is False
+ assert intent.authority.may_run_blocks is False
+ assert TurnIntentReasonCode.AUTHORING_INTENT_DEFER_AUTHORING in intent.reason_codes
+
+
+def test_build_turn_intent_defer_authoring_raw_secret_drops_reason_and_keeps_refusal() -> None:
+ policy = RequestPolicy(
+ authoring_intent="defer_authoring",
+ credential_input_kind="raw_secret",
+ raw_secret_detected=True,
+ user_response_policy="ask_clarification",
+ allow_update_workflow=False,
+ allow_run_blocks=False,
+ clarification_question="Store the credential in the Credentials UI.",
+ )
+
+ intent = build_turn_intent(
+ user_message="Fill the page now with password: hunter2, don't author a workflow yet.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=policy,
+ classifier_result=_classification(TurnIntentMode.BUILD),
+ )
+
+ assert intent.mode == TurnIntentMode.REFUSE
+ assert intent.authority.requires_user_input is True
+ assert TurnIntentReasonCode.RAW_SECRET_REFUSAL in intent.reason_codes
+ assert TurnIntentReasonCode.AUTHORING_INTENT_DEFER_AUTHORING not in intent.reason_codes
+ assert turn_intent_defers_authoring_live_fill(intent) is False
+
+
+def test_turn_intent_defers_authoring_live_fill_false_for_ordinary_build() -> None:
+ intent = build_turn_intent(
+ user_message="Build a workflow that fills the form and run it.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ request_policy=RequestPolicy(allow_update_workflow=True, allow_run_blocks=True),
+ classifier_result=_classification(TurnIntentMode.BUILD),
+ )
+
+ assert turn_intent_defers_authoring_live_fill(intent) is False
+
+
def test_build_turn_intent_redacts_user_goal() -> None:
intent = build_turn_intent(
user_message="Use password: hunter2 and build the workflow",
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index 030304a9c..b722d7efb 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -1452,6 +1452,9 @@ async def test_handle_action_uses_xhr_staging_fallback_when_no_native_file(
with (
patch.object(ActionHandler, "_handle_action", side_effect=mock_inner),
+ # No native file lands in the observed dir, so the wait loop would burn
+ # the full no-signal grace before falling back to xhr staging; shorten it.
+ patch("skyvern.webeye.actions.handler.BROWSER_DOWNLOAD_NO_SIGNAL_GRACE_TIME", 0.01),
patch("skyvern.webeye.actions.handler.get_download_dir", return_value=primary_dir),
patch("skyvern.webeye.actions.handler.make_temp_directory", return_value=staging),
patch(
@@ -1511,6 +1514,9 @@ async def test_handle_action_moves_multiple_staged_xhr_files_as_fallback(
with (
patch.object(ActionHandler, "_handle_action", side_effect=mock_inner),
+ # No native file lands in the observed dir, so the wait loop would burn
+ # the full no-signal grace before falling back to xhr staging; shorten it.
+ patch("skyvern.webeye.actions.handler.BROWSER_DOWNLOAD_NO_SIGNAL_GRACE_TIME", 0.01),
patch("skyvern.webeye.actions.handler.get_download_dir", return_value=primary_dir),
patch("skyvern.webeye.actions.handler.make_temp_directory", return_value=staging),
patch(
diff --git a/tests/unit/test_prompt_truncation.py b/tests/unit/test_prompt_truncation.py
index c61671f02..8dea068e3 100644
--- a/tests/unit/test_prompt_truncation.py
+++ b/tests/unit/test_prompt_truncation.py
@@ -54,7 +54,10 @@ def test_truncate_dict_preserves_top_level_keys_and_caps_values() -> None:
from skyvern.utils.prompt_truncation import truncate_previous_extracted_information
from skyvern.utils.token_counter import count_tokens
- value = {"a": "x" * 50_000, "b": "y" * 50_000}
+ # Each value only needs to overshoot the per-key budget (200 // 2 = 100 tokens)
+ # to exercise cropping; a handful of hundred tokens does that without paying to
+ # tokenize 100KB of filler.
+ value = {"a": "x" * 2_000, "b": "y" * 2_000}
result = truncate_previous_extracted_information(value, max_tokens=200)
assert isinstance(result, dict)
assert set(result.keys()) == {"a", "b"}
diff --git a/tests/unit/test_real_browser_state_navigate.py b/tests/unit/test_real_browser_state_navigate.py
index ff4cf1d4a..3394ab247 100644
--- a/tests/unit/test_real_browser_state_navigate.py
+++ b/tests/unit/test_real_browser_state_navigate.py
@@ -20,6 +20,14 @@ def browser_state() -> RealBrowserState:
return state
[email protected](autouse=True)
+def _instant_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
+ # These tests assert on strategy order and exceptions, not wall-clock timing.
+ # Collapse the retry backoff and the post-navigation settle to run instantly.
+ monkeypatch.setattr("skyvern.webeye.navigation.asyncio.sleep", AsyncMock())
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.asyncio.sleep", AsyncMock())
+
+
def test_same_page_ignoring_fragment_matches_fragment_only_differences() -> None:
assert _same_page_ignoring_fragment("https://example.test/results#section", "https://example.test/results") is True
assert _same_page_ignoring_fragment("https://example.test/results/", "https://example.test/results") is True
diff --git a/tests/unit/test_tag_endpoints.py b/tests/unit/test_tag_endpoints.py
index ac90581d6..9d1b629f9 100644
--- a/tests/unit/test_tag_endpoints.py
+++ b/tests/unit/test_tag_endpoints.py
@@ -13,7 +13,7 @@ of tag objects rather than key-maps.
from __future__ import annotations
import datetime as dt
-from typing import Any, AsyncGenerator
+from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -21,10 +21,10 @@ import pytest_asyncio
from fastapi import FastAPI
from fastapi.testclient import TestClient
from sqlalchemy.exc import IntegrityError
-from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine
+from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker
from skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDB
-from skyvern.forge.sdk.db.models import Base, WorkflowModel, WorkflowRunModel
+from skyvern.forge.sdk.db.models import WorkflowModel, WorkflowRunModel
from skyvern.forge.sdk.db.repositories.tags import TagsRepository
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.services import org_auth_service
@@ -50,13 +50,10 @@ def _by_key(tags: list[dict[str, Any]], key: str | None) -> dict[str, Any]:
@pytest_asyncio.fixture
-async def engine() -> AsyncGenerator[AsyncEngine]:
- eng = create_async_engine("sqlite+aiosqlite:///:memory:")
- async with eng.begin() as conn:
- await conn.run_sync(Base.metadata.create_all)
+async def engine(sqlite_engine: AsyncEngine) -> AsyncEngine:
# The workflow the tag routes operate on must exist (non-deleted) so the
# value-count filter can resolve it, mirroring the existence mock below.
- session_maker = async_sessionmaker(eng, expire_on_commit=False)
+ session_maker = async_sessionmaker(sqlite_engine, expire_on_commit=False)
async with session_maker() as session:
session.add(
WorkflowModel(organization_id=ORG_ID, workflow_permanent_id=WPID, title="t", workflow_definition={})
@@ -80,8 +77,7 @@ async def engine() -> AsyncGenerator[AsyncEngine]:
)
)
await session.commit()
- yield eng
- await eng.dispose()
+ return sqlite_engine
@pytest_asyncio.fixture
@@ -100,15 +96,16 @@ def _make_org(org_id: str = ORG_ID) -> Organization:
)
-def app_with_routes(repo: TagsRepository, monkeypatch: pytest.MonkeyPatch) -> FastAPI:
- """Mount the real routers with patched ``app`` module + dependency overrides.
[email protected](scope="module")
+def _route_app() -> tuple[FastAPI, MagicMock]:
+ """Mount the real routers once per module and return ``(app, app_mock)``.
- The route handlers reach the DB via ``app.DATABASE.tags`` and
- ``app.DATABASE.workflows``; we point both at fakes that delegate to the
- in-memory ``TagsRepository`` and a mock workflow lookup respectively.
+ ``include_router`` over the full ``base_router`` (~285 routes) costs ~0.3s, so
+ the app, the static workflow/run fakes, and the dependency overrides are built
+ once and shared. The route handlers reach the DB via ``app.DATABASE.tags`` and
+ ``app.DATABASE.workflows``; only the per-test ``TagsRepository`` and the tagging
+ flag are re-pointed in ``app_with_routes``.
"""
- from skyvern.forge.sdk.routes import agent_protocol as ap
from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router
workflows_mock = MagicMock()
@@ -166,15 +163,11 @@ def app_with_routes(repo: TagsRepository, monkeypatch: pytest.MonkeyPatch) -> Fa
workflow_runs_mock.get_workflow_runs_by_ids = AsyncMock(side_effect=_get_workflow_runs_by_ids)
database_mock = MagicMock()
- database_mock.tags = repo
database_mock.workflows = workflows_mock
database_mock.workflow_runs = workflow_runs_mock
app_mock = MagicMock()
app_mock.DATABASE = database_mock
- app_mock.AGENT_FUNCTION.is_workflow_tagging_enabled = AsyncMock(return_value=True)
-
- monkeypatch.setattr(ap, "app", app_mock)
test_app = FastAPI()
test_app.include_router(base_router, prefix="/v1")
@@ -193,6 +186,20 @@ def app_with_routes(repo: TagsRepository, monkeypatch: pytest.MonkeyPatch) -> Fa
test_app.dependency_overrides[org_auth_service.get_current_org] = _override_get_current_org
test_app.dependency_overrides[org_auth_service.get_current_caller_context] = _override_get_current_caller_context
+ return test_app, app_mock
+
+
+def app_with_routes(
+ repo: TagsRepository, monkeypatch: pytest.MonkeyPatch, _route_app: tuple[FastAPI, MagicMock]
+) -> FastAPI:
+ """Re-point the module-scoped app at this test's repo and reset the tagging flag."""
+ from skyvern.forge.sdk.routes import agent_protocol as ap
+
+ test_app, app_mock = _route_app
+ app_mock.DATABASE.tags = repo
+ app_mock.AGENT_FUNCTION.is_workflow_tagging_enabled = AsyncMock(return_value=True)
+ monkeypatch.setattr(ap, "app", app_mock)
return test_app
diff --git a/tests/unit/test_tags_repository.py b/tests/unit/test_tags_repository.py
index 6f56c8a7c..4448a169a 100644
--- a/tests/unit/test_tags_repository.py
+++ b/tests/unit/test_tags_repository.py
@@ -3,16 +3,14 @@
from __future__ import annotations
from datetime import datetime, timezone
-from typing import AsyncGenerator
import pytest
import pytest_asyncio
from sqlalchemy import func, select
-from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+from sqlalchemy.ext.asyncio import AsyncEngine
from skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDB
from skyvern.forge.sdk.db.models import (
- Base,
TagKeyModel,
TagValueModel,
WorkflowModel,
@@ -43,12 +41,8 @@ WRID = "wr_alpha"
@pytest_asyncio.fixture
-async def engine() -> AsyncGenerator[AsyncEngine]:
- eng = create_async_engine("sqlite+aiosqlite:///:memory:")
- async with eng.begin() as conn:
- await conn.run_sync(Base.metadata.create_all)
- yield eng
- await eng.dispose()
+async def engine(sqlite_engine: AsyncEngine) -> AsyncEngine:
+ return sqlite_engine
@pytest_asyncio.fixture
diff --git a/tests/unit/test_workflows_list_tag_filter.py b/tests/unit/test_workflows_list_tag_filter.py
index 7d091799b..d213fe31f 100644
--- a/tests/unit/test_workflows_list_tag_filter.py
+++ b/tests/unit/test_workflows_list_tag_filter.py
@@ -12,17 +12,16 @@ Two layers:
from __future__ import annotations
from datetime import datetime, timezone
-from typing import AsyncGenerator
from unittest.mock import AsyncMock, MagicMock
import pytest
import pytest_asyncio
from fastapi import FastAPI
from fastapi.testclient import TestClient
-from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+from sqlalchemy.ext.asyncio import AsyncEngine
from skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDB
-from skyvern.forge.sdk.db.models import Base, WorkflowModel, WorkflowTagEventModel
+from skyvern.forge.sdk.db.models import WorkflowModel, WorkflowTagEventModel
from skyvern.forge.sdk.db.repositories.workflows import WorkflowsRepository
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.services import org_auth_service
@@ -33,12 +32,8 @@ OTHER_ORG_ID = "o_other"
@pytest_asyncio.fixture
-async def engine() -> AsyncGenerator[AsyncEngine]:
- eng = create_async_engine("sqlite+aiosqlite:///:memory:")
- async with eng.begin() as conn:
- await conn.run_sync(Base.metadata.create_all)
- yield eng
- await eng.dispose()
+async def engine(sqlite_engine: AsyncEngine) -> AsyncEngine:
+ return sqlite_engine
@pytest_asyncio.fixture
@@ -288,16 +283,17 @@ def _make_org(org_id: str = ORG_ID) -> Organization:
)
-def route_client(monkeypatch: pytest.MonkeyPatch) -> TestClient:
- """Mount the real GET /workflows route with a mocked WORKFLOW_SERVICE.
[email protected](scope="module")
+def _route_app() -> tuple[FastAPI, MagicMock, dict[str, object]]:
+ """Mount the real GET /workflows route once per module with a mocked WORKFLOW_SERVICE.
Captures the workflow_tags value the route passes to the service so the
?tags= registration + key:value parsing (including 400s) are tested end to
end. Importing the router module also guards against the route-registration
failure caught in debate round 1 — a bad param declaration raises at import.
+ ``include_router`` is expensive, so the app is shared; ``route_client`` clears
+ the captured dict and re-applies the ``app`` monkeypatch per test.
"""
- from skyvern.forge.sdk.routes import agent_protocol as ap
from skyvern.forge.sdk.routes.routers import base_router
captured: dict[str, object] = {}
@@ -314,12 +310,24 @@ def route_client(monkeypatch: pytest.MonkeyPatch) -> TestClient:
# template=true path: empty global list short-circuits to [] before any service call.
app_mock.STORAGE.retrieve_global_workflows = AsyncMock(return_value=[])
app_mock.AGENT_FUNCTION.is_workflow_tagging_enabled = AsyncMock(return_value=True)
- monkeypatch.setattr(ap, "app", app_mock)
test_app = FastAPI()
test_app.include_router(base_router, prefix="/v1")
test_app.dependency_overrides[org_auth_service.get_current_org] = lambda: _make_org(ORG_ID)
+ return test_app, app_mock, captured
+
+
+def route_client(
+ monkeypatch: pytest.MonkeyPatch, _route_app: tuple[FastAPI, MagicMock, dict[str, object]]
+) -> TestClient:
+ test_app, app_mock, captured = _route_app
+ captured.clear()
+ from skyvern.forge.sdk.routes import agent_protocol as ap
+
+ monkeypatch.setattr(ap, "app", app_mock)
+
client = TestClient(test_app)
client.captured = captured # type: ignore[attr-defined]
return client