Skip to content

Automate browser based workflows with AI

Stacked: cross-process PBS reads (SKY-13473), fail-closed script execution, vendor teardown budget (#8032)

Aaron Perezb416fd48aef1
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 0a15af25f..85cf853f8 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -1493,6 +1493,13 @@ class ScriptTerminationException(SkyvernException):
super().__init__(reason)
+class InProcessScriptExecutionDenied(SkyvernException):
+ def __init__(self, *, seam: str, selection_reason: str) -> None:
+ self.seam = seam
+ self.selection_reason = selection_reason
+ super().__init__(f"In-process script execution denied at {seam}: {selection_reason}")
+
+
class IllegitCompleteScriptTermination(ScriptTerminationException):
"""Raised when a cached script's page.complete() is rejected by the verifier; distinct from plain ScriptTerminationException, which is an intentional terminate()."""
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 78c928cb8..65e80b586 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -183,6 +183,14 @@ class CopilotSiteOriginAssociation:
provider_relation_text: str
+@dataclass(frozen=True)
+class ScriptExecutionPolicyDecision:
+ allowed: bool
+ selection_reason: str
+ flag_value: bool | None = None
+ env_force_on: bool = False
+
+
class CopilotCandidateNetworkHop(TypedDict):
url: str
resource_type: str
@@ -1040,6 +1048,17 @@ class AgentFunction:
"""
return False
+ async def resolve_in_process_script_execution_policy(
+ self,
+ *,
+ organization_id: str | None,
+ workflow_run_id: str | None,
+ workflow_permanent_id: str | None = None,
+ workflow_id: str | None = None,
+ script_id: str | None = None,
+ ) -> ScriptExecutionPolicyDecision:
+ return ScriptExecutionPolicyDecision(allowed=True, selection_reason="oss_default")
+
async def should_auto_create_browser_session_for_code_block(
self,
*,
diff --git a/skyvern/forge/sdk/experimentation/providers.py b/skyvern/forge/sdk/experimentation/providers.py
index 25a4ad179..686fbbaab 100644
--- a/skyvern/forge/sdk/experimentation/providers.py
+++ b/skyvern/forge/sdk/experimentation/providers.py
@@ -170,6 +170,11 @@ class BaseExperimentationProvider(ABC):
self.variant_map: TTLCache = TTLCache(maxsize=EXPERIMENTATION_CACHE_MAX_SIZE, ttl=EXPERIMENTATION_CACHE_TTL)
self.payload_map: TTLCache = TTLCache(maxsize=EXPERIMENTATION_CACHE_MAX_SIZE, ttl=EXPERIMENTATION_CACHE_TTL)
self.tri_state_map: TTLCache = TTLCache(maxsize=EXPERIMENTATION_CACHE_MAX_SIZE, ttl=EXPERIMENTATION_CACHE_TTL)
+ # Kept apart from tri_state_map: a lenient resolution caches a swallowed evaluation error
+ # as None, and serving that to a strict caller would silently break its no-swallow contract.
+ self.strict_tri_state_map: TTLCache = TTLCache(
+ maxsize=EXPERIMENTATION_CACHE_MAX_SIZE, ttl=EXPERIMENTATION_CACHE_TTL
+ )
@abstractmethod
async def _is_feature_enabled(self, feature_name: str, distinct_id: str, properties: dict | None = None) -> bool:
@@ -180,9 +185,16 @@ class BaseExperimentationProvider(ABC):
def invalidate_resolution_caches(self) -> int:
"""Drop all cached resolutions; returns how many entries were dropped."""
- dropped = len(self.result_map) + len(self.tri_state_map) + len(self.variant_map) + len(self.payload_map)
+ dropped = (
+ len(self.result_map)
+ + len(self.tri_state_map)
+ + len(self.strict_tri_state_map)
+ + len(self.variant_map)
+ + len(self.payload_map)
+ )
self.result_map.clear()
self.tri_state_map.clear()
+ self.strict_tri_state_map.clear()
self.variant_map.clear()
self.payload_map.clear()
return dropped
@@ -240,6 +252,33 @@ class BaseExperimentationProvider(ABC):
"""
return await self._is_feature_enabled(feature_name, distinct_id, properties)
+ async def _resolve_feature_flag_strict(
+ self, feature_name: str, distinct_id: str, properties: dict | None = None
+ ) -> bool | None:
+ return await self._resolve_feature_flag(feature_name, distinct_id, properties)
+
+ async def resolve_feature_flag_strict(
+ self, feature_name: str, distinct_id: str, properties: dict | None = None
+ ) -> bool | None:
+ """Resolve a tri-state flag without allowing providers to swallow evaluation errors."""
+ cache_key = _make_cache_key(feature_name, distinct_id, properties)
+ if should_bypass_feature_flag_cache(feature_name):
+ await self._prepare_feature_flag_resolution(feature_name, cached=False)
+ resolved = await self._resolve_feature_flag_strict(feature_name, distinct_id, properties)
+ else:
+ await self._prepare_feature_flag_resolution(feature_name, cached=True)
+ if cache_key in self.strict_tri_state_map:
+ resolved = self.strict_tri_state_map[cache_key]
+ else:
+ resolved = await self._resolve_feature_flag_strict(feature_name, distinct_id, properties)
+ self.strict_tri_state_map[cache_key] = resolved
+ record_feature_flag_resolution(
+ feature_name=feature_name,
+ resolution_kind="enabled",
+ resolved_value=resolved,
+ )
+ return resolved
+
async def resolve_feature_flag_cached(
self, feature_name: str, distinct_id: str, properties: dict | None = None
) -> bool | None:
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index a814a9411..37dea619c 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -45,6 +45,7 @@ from skyvern.exceptions import (
BrowserSessionNotFound,
BrowserSessionNotRenewable,
DisabledBlockExecutionError,
+ InProcessScriptExecutionDenied,
InvalidCredentialId,
InvalidWorkflowParameter,
MissingValueForParameter,
@@ -626,7 +627,17 @@ def _script_has_static_module_marker(script_path: str) -> bool:
return False
-def _load_user_script_module(script_path: str, spec: "importlib.machinery.ModuleSpec") -> Any | None:
+async def _load_user_script_module(
+ script_path: str,
+ spec: "importlib.machinery.ModuleSpec",
+ *,
+ organization_id: str | None = None,
+ workflow_run_id: str | None = None,
+ workflow_permanent_id: str | None = None,
+ workflow_id: str | None = None,
+ script_id: str | None = None,
+ script_revision_id: str | None = None,
+) -> Any | None:
"""Load a cached script's main.py for execution.
Static (marker) pins must import the live platform module FIRST: the stored
@@ -639,6 +650,16 @@ def _load_user_script_module(script_path: str, spec: "importlib.machinery.Module
check would never supersede it. Only a markerless pin (a generated script)
executes its stored body.
"""
+ await script_service.ensure_in_process_script_execution_allowed(
+ seam="workflow.cached_script_module_load",
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ workflow_id=workflow_id,
+ script_id=script_id,
+ script_revision_id=script_revision_id,
+ )
+
loaded_script_module = app.AGENT_FUNCTION.try_import_static_script(script_path)
if loaded_script_module is not None:
return loaded_script_module
@@ -4100,7 +4121,16 @@ class WorkflowService:
spec = importlib.util.spec_from_file_location("user_script", script_path)
if spec and spec.loader:
- loaded_script_module = _load_user_script_module(script_path, spec)
+ loaded_script_module = await _load_user_script_module(
+ script_path,
+ spec,
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_id=workflow.workflow_id,
+ script_id=script.script_id,
+ script_revision_id=script.script_revision_id,
+ )
param_cls = (
getattr(loaded_script_module, "GeneratedWorkflowParameters", None)
if loaded_script_module
@@ -4158,6 +4188,8 @@ class WorkflowService:
script_path=script_path,
script_id=script.script_id,
)
+ except InProcessScriptExecutionDenied:
+ raise
except Exception as e:
LOG.warning(
"Failed to load script blocks, will fallback to normal execution",
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index e8cc09f54..f3d6ec62c 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -28,7 +28,9 @@ from skyvern.core.script_generations.script_skyvern_page import script_run_conte
from skyvern.errors.errors import UserDefinedError, filter_to_user_defined_codes
from skyvern.exceptions import (
CachedDownloadError,
+ CodeBlockRunnerSelectionError,
IllegitCompleteScriptTermination,
+ InProcessScriptExecutionDenied,
ScriptNotFound,
ScriptTerminationException,
StepTerminationError,
@@ -2955,6 +2957,64 @@ async def wait(seconds: int, label: str | None = None) -> None:
raise
+async def ensure_in_process_script_execution_allowed(
+ *,
+ seam: str,
+ organization_id: str | None,
+ workflow_run_id: str | None,
+ workflow_permanent_id: str | None = None,
+ workflow_id: str | None = None,
+ script_id: str | None = None,
+ script_revision_id: str | None = None,
+) -> None:
+ try:
+ decision = await app.AGENT_FUNCTION.resolve_in_process_script_execution_policy(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ workflow_id=workflow_id,
+ script_id=script_id,
+ )
+ except CodeBlockRunnerSelectionError as exc:
+ LOG.error(
+ "script.in_process_execution_denied",
+ seam=seam,
+ selection_reason="policy_evaluation_error",
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ workflow_id=workflow_id,
+ script_id=script_id,
+ script_revision_id=script_revision_id,
+ exc_info=True,
+ )
+ raise InProcessScriptExecutionDenied(
+ seam=seam,
+ selection_reason="policy_evaluation_error",
+ ) from exc
+
+ if decision.allowed:
+ return
+
+ LOG.error(
+ "script.in_process_execution_denied",
+ seam=seam,
+ selection_reason=decision.selection_reason,
+ flag_value=decision.flag_value,
+ env_force_on=decision.env_force_on,
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ workflow_permanent_id=workflow_permanent_id,
+ workflow_id=workflow_id,
+ script_id=script_id,
+ script_revision_id=script_revision_id,
+ )
+ raise InProcessScriptExecutionDenied(
+ seam=seam,
+ selection_reason=decision.selection_reason,
+ )
+
+
async def run_script(
path: str,
parameters: dict[str, Any] | None = None,
@@ -2994,8 +3054,14 @@ async def run_script(
context.workflow_run_id = workflow_run_id
context.organization_id = organization_id
- # run the script as subprocess; pass the parameters and run_id to the script
- # Dynamically import the script at the given path
+ await ensure_in_process_script_execution_allowed(
+ seam="script_service.run_script",
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ script_id=script_id,
+ script_revision_id=script_revision_id,
+ )
+
spec = importlib.util.spec_from_file_location("user_script", path)
if not spec or not spec.loader:
raise Exception(f"Failed to import script from {path}")