Skip to content

Automate browser based workflows with AI

feat: add use_in_memory_db to Skyvern.local() (#5260)

Marc Kelechava466551eabfa0
diff --git a/pyproject.toml b/pyproject.toml
index 5d6918097..e9ba25c1f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -77,6 +77,7 @@ dependencies = [
"pyasn1>=0.6.3",
"PyJWT>=2.12.0",
"tornado>=6.5.5",
+ "aiosqlite>=0.21.0,<0.22",
]
[dependency-groups]
diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.py
index 731682f26..696ba03cc 100644
--- a/skyvern/forge/api_app.py
+++ b/skyvern/forge/api_app.py
@@ -25,11 +25,6 @@ from skyvern.forge.sdk.routes import internal_auth
from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router, legacy_v2_router
from skyvern.services.cleanup_service import start_cleanup_scheduler, stop_cleanup_scheduler
-try:
- from cloud.observability.otel_setup import OTELSetup
-except ImportError:
- OTELSetup = None # type: ignore[assignment,misc]
-
LOG = structlog.get_logger()
@@ -131,13 +126,23 @@ def create_api_app() -> FastAPI:
"""
Start the agent server.
"""
- # CRITICAL: Initialize OTEL FIRST, before any other code runs
+ # CRITICAL: Initialize OTEL FIRST, before any other code runs.
# This must happen before start_forge_app() because that function
# creates database connections. If we don't instrument the libraries
# first, the DB spans won't be children of the HTTP request spans.
- if settings.OTEL_ENABLED and OTELSetup is not None:
+ # NOTE: The cloud import is lazy (not at module level) to avoid triggering
+ # cloud/__init__.py side effects when skyvern is used in embedded mode.
+ otel_setup_cls: Any = None
+ if settings.OTEL_ENABLED:
+ try:
+ from cloud.observability.otel_setup import OTELSetup # noqa: PLC0415
+
+ otel_setup_cls = OTELSetup
+ except ImportError:
+ pass
+ if otel_setup_cls is not None:
try:
- otel = OTELSetup.get_instance()
+ otel = otel_setup_cls.get_instance()
otel.initialize_tracer_provider()
LOG.info("OTEL tracer provider initialized before forge app creation")
except Exception as e:
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index 2322a70e6..be84adbf5 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -34,6 +34,11 @@ class LLMConfigRegistry:
LOG.debug("Registering LLM config", llm_key=llm_key)
cls._configs[llm_key] = config
+ @classmethod
+ def deregister_config(cls, llm_key: str) -> None:
+ """Remove a registered LLM config. Idempotent — no-op if key doesn't exist."""
+ cls._configs.pop(llm_key, None)
+
@classmethod
def get_config(cls, llm_key: str) -> LLMRouterConfig | LLMConfig:
if llm_key not in cls._configs:
diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.py
index 91c343a93..edd2537aa 100644
--- a/skyvern/forge/sdk/db/agent_db.py
+++ b/skyvern/forge/sdk/db/agent_db.py
@@ -1,3 +1,4 @@
+import asyncio
import json
import uuid
from datetime import UTC, datetime, timedelta
@@ -12,21 +13,24 @@ from sqlalchemy import (
cast,
delete,
distinct,
+ event,
exists,
func,
literal,
+ literal_column,
or_,
pool,
select,
text,
- tuple_,
update,
)
-from sqlalchemy.dialects.postgresql import JSONB, insert
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
from sqlalchemy.exc import (
SQLAlchemyError,
)
-from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, create_async_engine
from skyvern.config import settings
from skyvern.constants import DEFAULT_SCRIPT_RUN_ID
@@ -206,17 +210,45 @@ def _serialize_proxy_location(proxy_location: ProxyLocationInput) -> str | None:
return result
-def _build_engine(database_string: str) -> Any:
+def _build_engine(database_string: str) -> AsyncEngine:
"""
Build a SQLAlchemy async engine.
- When DISABLE_CONNECTION_POOL=True (NullPool): enforce statement_timeout
- and allow prepared statements.
+ Supports both PostgreSQL and SQLite (via aiosqlite) dialects.
- When DISABLE_CONNECTION_POOL=False (QueuePool): disable prepared statements
- and do not set statement_timeout - set at role level in the database,
- since the transaction pooler does not maintain session-level settings.
+ PostgreSQL behaviour:
+ When DISABLE_CONNECTION_POOL=True (NullPool): enforce statement_timeout
+ and allow prepared statements.
+ When DISABLE_CONNECTION_POOL=False (QueuePool): disable prepared statements
+ and do not set statement_timeout - set at role level in the database,
+ since the transaction pooler does not maintain session-level settings.
+
+ SQLite behaviour:
+ For :memory: databases, uses StaticPool to keep the single connection alive.
+ For file-backed databases, enables WAL mode for concurrent read support.
+ Always enables foreign key enforcement via PRAGMA.
"""
+ if database_string.startswith("sqlite"):
+ is_memory = ":memory:" in database_string
+ engine_kwargs: dict[str, Any] = {
+ "json_serializer": _custom_json_serializer,
+ }
+ if is_memory:
+ engine_kwargs["poolclass"] = pool.StaticPool
+ engine_kwargs["connect_args"] = {"check_same_thread": False}
+ engine = create_async_engine(database_string, **engine_kwargs)
+
+ @event.listens_for(engine.sync_engine, "connect")
+ def _set_sqlite_pragmas(dbapi_conn: Any, connection_record: Any) -> None:
+ cursor = dbapi_conn.cursor()
+ cursor.execute("PRAGMA foreign_keys=ON")
+ if not is_memory:
+ cursor.execute("PRAGMA journal_mode=WAL")
+ cursor.close()
+
+ return engine
+
+ # PostgreSQL path (unchanged)
connect_args: dict[str, Any] = {}
if settings.DISABLE_CONNECTION_POOL:
if "postgresql+psycopg" in database_string:
@@ -260,6 +292,12 @@ class AgentDB(BaseAlchemyDB):
def __init__(self, database_string: str, debug_enabled: bool = False, db_engine: AsyncEngine | None = None) -> None:
super().__init__(db_engine or _build_engine(database_string))
self.debug_enabled = debug_enabled
+ # Global lock for SQLite schedule serialization. Unlike Postgres advisory locks
+ # (which are scoped per org:workflow via hashtext(key)), this serializes ALL
+ # schedule creates across all workflows. Acceptable for single-user embedded mode.
+ self._sqlite_schedule_lock: asyncio.Lock | None = (
+ asyncio.Lock() if self.engine.dialect.name == "sqlite" else None
+ )
def is_retryable_error(self, error: SQLAlchemyError) -> bool:
error_msg = str(error).lower()
@@ -564,11 +602,14 @@ class AgentDB(BaseAlchemyDB):
"""
try:
async with self.Session() as session:
- query = (
- select(func.count(distinct(tuple_(StepModel.task_id, StepModel.order))))
+ subq = (
+ select(StepModel.task_id, StepModel.order)
.where(StepModel.task_id.in_(task_ids))
.where(StepModel.organization_id == organization_id)
+ .distinct()
+ .subquery()
)
+ query = select(func.count()).select_from(subq)
return (await session.execute(query)).scalar()
except SQLAlchemyError:
LOG.error("SQLAlchemyError", exc_info=True)
@@ -2131,52 +2172,61 @@ class AgentDB(BaseAlchemyDB):
) -> tuple[WorkflowSchedule, int]:
"""Create a schedule atomically with limit enforcement.
- Uses a PostgreSQL advisory lock to serialize concurrent creates for the
- same workflow, preventing TOCTOU races on the schedule count.
+ Uses a PostgreSQL advisory lock (or asyncio.Lock for SQLite) to serialize
+ concurrent creates for the same workflow, preventing TOCTOU races on the
+ schedule count.
Returns (created_schedule, count_before_insert).
Raises ScheduleLimitExceededError if count >= max_schedules.
"""
- try:
- async with self.Session() as session:
- lock_key = f"schedule:{organization_id}:{workflow_permanent_id}"
- await session.execute(
- text("SELECT pg_advisory_xact_lock(hashtext(:key))"),
- {"key": lock_key},
- )
- count = (
- await session.execute(
- select(func.count()).where(
- WorkflowScheduleModel.organization_id == organization_id,
- WorkflowScheduleModel.workflow_permanent_id == workflow_permanent_id,
- WorkflowScheduleModel.deleted_at.is_(None),
- )
- )
- ).scalar_one()
-
- if max_schedules is not None and count >= max_schedules:
- raise ScheduleLimitExceededError(
- organization_id=organization_id,
- workflow_permanent_id=workflow_permanent_id,
- current_count=count,
- max_allowed=max_schedules,
+ async def _insert(session: AsyncSession) -> tuple[WorkflowSchedule, int]:
+ count = (
+ await session.execute(
+ select(func.count()).where(
+ WorkflowScheduleModel.organization_id == organization_id,
+ WorkflowScheduleModel.workflow_permanent_id == workflow_permanent_id,
+ WorkflowScheduleModel.deleted_at.is_(None),
)
+ )
+ ).scalar_one()
- workflow_schedule = WorkflowScheduleModel(
+ if max_schedules is not None and count >= max_schedules:
+ raise ScheduleLimitExceededError(
organization_id=organization_id,
workflow_permanent_id=workflow_permanent_id,
- cron_expression=cron_expression,
- timezone=timezone,
- enabled=enabled,
- parameters=parameters,
- name=name,
- description=description,
+ current_count=count,
+ max_allowed=max_schedules,
)
- session.add(workflow_schedule)
- await session.commit()
- await session.refresh(workflow_schedule)
- return convert_to_workflow_schedule(workflow_schedule, self.debug_enabled), count
+
+ workflow_schedule = WorkflowScheduleModel(
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ cron_expression=cron_expression,
+ timezone=timezone,
+ enabled=enabled,
+ parameters=parameters,
+ name=name,
+ description=description,
+ )
+ session.add(workflow_schedule)
+ await session.commit()
+ await session.refresh(workflow_schedule)
+ return convert_to_workflow_schedule(workflow_schedule, self.debug_enabled), count
+
+ try:
+ if self._sqlite_schedule_lock:
+ async with self._sqlite_schedule_lock:
+ async with self.Session() as session:
+ return await _insert(session)
+ else:
+ async with self.Session() as session:
+ lock_key = f"schedule:{organization_id}:{workflow_permanent_id}"
+ await session.execute(
+ text("SELECT pg_advisory_xact_lock(hashtext(:key))"),
+ {"key": lock_key},
+ )
+ return await _insert(session)
except SQLAlchemyError:
LOG.error("SQLAlchemyError", exc_info=True)
raise
@@ -4043,22 +4093,54 @@ class AgentDB(BaseAlchemyDB):
)
return query.where(or_(id_matches, param_key_desc_exists, param_value_exists, extra_headers_match))
- @staticmethod
- def _apply_error_code_filter(query, error_code: str | None): # type: ignore[no-untyped-def]
+ def _apply_error_code_filter(self, query, error_code: str | None): # type: ignore[no-untyped-def]
if not error_code:
return query
- error_code_in_tasks = exists(
- select(1)
- .select_from(TaskModel)
- .where(TaskModel.workflow_run_id == WorkflowRunModel.workflow_run_id)
- .where(cast(TaskModel.errors, JSONB).contains(literal([{"error_code": error_code}], type_=JSONB)))
- )
- error_code_in_blocks = exists(
- select(1)
- .select_from(WorkflowRunBlockModel)
- .where(WorkflowRunBlockModel.workflow_run_id == WorkflowRunModel.workflow_run_id)
- .where(cast(WorkflowRunBlockModel.error_codes, JSONB).contains(literal([error_code], type_=JSONB)))
- )
+
+ dialect_name = self.engine.dialect.name
+
+ if dialect_name == "sqlite":
+ # Task errors: array of objects like [{"error_code": "timeout", ...}]
+ # Use json_each to iterate + json_extract to match the error_code field
+ error_code_in_tasks = exists(
+ select(1)
+ .select_from(TaskModel)
+ .where(TaskModel.workflow_run_id == WorkflowRunModel.workflow_run_id)
+ .where(
+ exists(
+ select(1)
+ .select_from(func.json_each(TaskModel.errors))
+ .where(func.json_extract(literal_column("json_each.value"), "$.error_code") == error_code)
+ )
+ )
+ )
+ # Block errors: flat array of strings like ["timeout", "network_error"]
+ error_code_in_blocks = exists(
+ select(1)
+ .select_from(WorkflowRunBlockModel)
+ .where(WorkflowRunBlockModel.workflow_run_id == WorkflowRunModel.workflow_run_id)
+ .where(
+ exists(
+ select(1)
+ .select_from(func.json_each(WorkflowRunBlockModel.error_codes))
+ .where(literal_column("json_each.value") == error_code)
+ )
+ )
+ )
+ else:
+ # PostgreSQL: native JSONB containment
+ error_code_in_tasks = exists(
+ select(1)
+ .select_from(TaskModel)
+ .where(TaskModel.workflow_run_id == WorkflowRunModel.workflow_run_id)
+ .where(cast(TaskModel.errors, JSONB).contains(literal([{"error_code": error_code}], type_=JSONB)))
+ )
+ error_code_in_blocks = exists(
+ select(1)
+ .select_from(WorkflowRunBlockModel)
+ .where(WorkflowRunBlockModel.workflow_run_id == WorkflowRunModel.workflow_run_id)
+ .where(cast(WorkflowRunBlockModel.error_codes, JSONB).contains(literal([error_code], type_=JSONB)))
+ )
return query.where(or_(error_code_in_tasks, error_code_in_blocks))
async def get_workflow_runs(
@@ -8007,7 +8089,9 @@ class AgentDB(BaseAlchemyDB):
now = datetime.utcnow()
try:
async with self.Session() as session:
- stmt = insert(ScriptBranchHitModel).values(
+ dialect_insert = sqlite_insert if self.engine.dialect.name == "sqlite" else pg_insert
+
+ stmt = dialect_insert(ScriptBranchHitModel).values(
organization_id=organization_id,
workflow_permanent_id=workflow_permanent_id,
block_label=block_label,
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 0f0cc9536..df6f570eb 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -249,7 +249,7 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)
And return the dumped workflow definition as a python dictionary.
"""
# Convert the workflow definition to a dictionary
- workflow_dict = workflow_definition.model_dump()
+ workflow_dict = workflow_definition.model_dump(mode="json")
fields_to_remove = [
"created_at",
"modified_at",
@@ -2835,7 +2835,7 @@ class WorkflowService:
try:
return await app.DATABASE.create_workflow(
title=title,
- workflow_definition=workflow_definition.model_dump(),
+ workflow_definition=workflow_definition.model_dump(mode="json"),
organization_id=organization_id,
description=description,
proxy_location=proxy_location,
@@ -3190,7 +3190,7 @@ class WorkflowService:
title=title,
organization_id=organization_id,
description=description,
- workflow_definition=(workflow_definition.model_dump() if workflow_definition else None),
+ workflow_definition=(workflow_definition.model_dump(mode="json") if workflow_definition else None),
)
return updated_workflow
diff --git a/skyvern/library/embedded_server_factory.py b/skyvern/library/embedded_server_factory.py
index 52060db7b..1562841ca 100644
--- a/skyvern/library/embedded_server_factory.py
+++ b/skyvern/library/embedded_server_factory.py
@@ -1,5 +1,8 @@
import os
+import tempfile
+from datetime import timedelta
from typing import Any
+from uuid import uuid4
import httpx
from httpx import ASGITransport
@@ -9,46 +12,196 @@ from skyvern.forge.api_app import create_api_app
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.models import LLMConfig, LLMRouterConfig
+_EMBEDDED_ACTIVE: bool = False
+
+_BLOCKED_EMBEDDED_SETTINGS = frozenset({"OTEL_ENABLED", "ENABLE_CLEANUP_CRON"})
+
def create_embedded_server(
llm_config: LLMRouterConfig | LLMConfig | None = None,
settings_overrides: dict[str, Any] | None = None,
+ use_in_memory_db: bool = False,
) -> httpx.AsyncClient:
+ global _EMBEDDED_ACTIVE
+ if _EMBEDDED_ACTIVE:
+ raise RuntimeError(
+ "An embedded Skyvern client is already active in this process. "
+ "Call aclose() on the existing client before creating a new one."
+ )
+ _EMBEDDED_ACTIVE = True
+
class EmbeddedServerTransport(httpx.AsyncBaseTransport):
def __init__(self) -> None:
self._transport: ASGITransport | None = None
self._api_key: str | None = None
+ self._artifact_dir: str | None = None
+ self._llm_key: str | None = None
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
if self._transport is None:
- settings.BROWSER_LOGS_ENABLED = False
+ try:
+ await self._bootstrap(request)
+ except Exception:
+ # Reset the single-client guard so the user can retry
+ # after fixing the issue (e.g., missing API key).
+ global _EMBEDDED_ACTIVE
+ _EMBEDDED_ACTIVE = False
+ raise
- if llm_config:
- LLMConfigRegistry.register_config(
- "CUSTOM_LLM",
- llm_config,
- )
- settings.LLM_KEY = "CUSTOM_LLM"
+ if self._api_key and "x-api-key" not in request.headers:
+ request.headers["x-api-key"] = self._api_key
+
+ assert self._transport is not None
+ response = await self._transport.handle_async_request(request)
+ return response
+
+ async def _bootstrap(self, request: httpx.Request) -> None:
+ """One-time lazy initialization of the embedded server."""
+ settings.BROWSER_LOGS_ENABLED = False
- # Apply custom settings overrides
- if settings_overrides:
- for key, value in settings_overrides.items():
- if hasattr(settings, key):
- setattr(settings, key, value)
- else:
- raise ValueError(f"Invalid setting: {key}")
+ # Register custom LLM BEFORE create_api_app — create_forge_app()
+ # reads settings.LLM_KEY to build handlers during startup.
+ if llm_config:
+ llm_key = f"CUSTOM_LLM_{uuid4().hex[:8]}"
+ LLMConfigRegistry.register_config(llm_key, llm_config)
+ settings.LLM_KEY = llm_key
+ self._llm_key = llm_key
+ # Validate and apply custom settings overrides
+ if settings_overrides:
+ blocked = set(settings_overrides) & _BLOCKED_EMBEDDED_SETTINGS
+ if blocked:
+ raise ValueError(
+ f"Cannot override {blocked} in embedded mode — "
+ f"these require FastAPI lifespan which is not available under ASGITransport."
+ )
+ for key, value in settings_overrides.items():
+ if hasattr(settings, key):
+ setattr(settings, key, value)
+ else:
+ raise ValueError(f"Invalid setting: {key}")
+
+ if not use_in_memory_db:
self._api_key = os.getenv("SKYVERN_API_KEY")
if not self._api_key:
raise ValueError("SKYVERN_API_KEY is not set. Provide api_key or set SKYVERN_API_KEY in .env file.")
- api_app = create_api_app()
- self._transport = ASGITransport(app=api_app)
+ if use_in_memory_db:
+ # Set overrides BEFORE create_api_app so that start_forge_app()
+ # sees ADDITIONAL_MODULES=[] and skips cloud module loading.
+ _apply_sqlite_overrides()
- if self._api_key and "x-api-key" not in request.headers:
- request.headers["x-api-key"] = self._api_key
+ api_app = create_api_app()
- response = await self._transport.handle_async_request(request)
- return response
+ if use_in_memory_db:
+ self._api_key = await self._bootstrap_in_memory_db()
+
+ self._transport = ASGITransport(app=api_app)
+
+ async def _bootstrap_in_memory_db(self) -> str:
+ """Create tables, org, and auth token for in-memory SQLite. Returns the API token."""
+ from skyvern.forge import app as forge_app # noqa: PLC0415
+ from skyvern.forge.sdk.artifact.storage.factory import StorageFactory # noqa: PLC0415
+ from skyvern.forge.sdk.artifact.storage.local import LocalStorage # noqa: PLC0415
+ from skyvern.forge.sdk.core.security import create_access_token # noqa: PLC0415
+ from skyvern.forge.sdk.db.agent_db import AgentDB # noqa: PLC0415
+ from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType # noqa: PLC0415
+ from skyvern.forge.sdk.db.models import Base # noqa: PLC0415
+
+ # Only replace the engine if cloud/__init__.py swapped it to Postgres.
+ # When _apply_sqlite_overrides() runs before create_api_app(), the engine
+ # should already be SQLite. Replacing it unconditionally would create a
+ # second :memory: database, discarding any state from app startup.
+ if forge_app.DATABASE.engine.dialect.name != "sqlite":
+ old_engine = forge_app.DATABASE.engine
+ forge_app.DATABASE = AgentDB("sqlite+aiosqlite:///:memory:")
+ forge_app.REPLICA_DATABASE = forge_app.DATABASE
+ await old_engine.dispose()
+ else:
+ forge_app.REPLICA_DATABASE = forge_app.DATABASE
+
+ # Rebuild LocalStorage with a temp directory for artifacts.
+ # LocalStorage.__init__ captures ARTIFACT_STORAGE_PATH at import time,
+ # so changing the setting later has no effect on the existing instance.
+ self._artifact_dir = tempfile.mkdtemp(prefix="skyvern-artifacts-")
+ StorageFactory.set_storage(LocalStorage(artifact_path=self._artifact_dir))
+ forge_app.STORAGE = StorageFactory.get_storage()
+
+ db = forge_app.DATABASE
+ async with db.engine.begin() as conn:
+ await conn.run_sync(Base.metadata.create_all)
+
+ org = await db.create_organization(organization_name="local")
+ token = create_access_token(org.organization_id, expires_delta=timedelta(days=365 * 10))
+ await db.create_org_auth_token(
+ organization_id=org.organization_id,
+ token_type=OrganizationAuthTokenType.api,
+ token=token,
+ )
+ return token
+
+ async def aclose(self) -> None:
+ """Dispose embedded resources: SQLite engine, artifact dir, and transport."""
+ global _EMBEDDED_ACTIVE
+ try:
+ from skyvern.forge import app as forge_app # noqa: PLC0415
+
+ if self._transport is not None and forge_app.DATABASE and forge_app.DATABASE.engine:
+ await forge_app.DATABASE.engine.dispose()
+ except Exception:
+ import structlog # noqa: PLC0415
+
+ structlog.get_logger().warning("Failed to dispose embedded engine", exc_info=True)
+
+ if self._llm_key:
+ LLMConfigRegistry.deregister_config(self._llm_key)
+ self._llm_key = None
+
+ if self._artifact_dir:
+ import shutil # noqa: PLC0415
+
+ shutil.rmtree(self._artifact_dir, ignore_errors=True)
+ self._artifact_dir = None
+
+ self._transport = None
+ self._api_key = None
+ _EMBEDDED_ACTIVE = False
+
+ def __del__(self) -> None:
+ """Fallback: reset single-client guard if aclose() was never called."""
+ global _EMBEDDED_ACTIVE
+ if _EMBEDDED_ACTIVE:
+ _EMBEDDED_ACTIVE = False
+
+ transport = EmbeddedServerTransport()
+ return EmbeddedClient(transport=transport, base_url="http://skyvern-embedded")
+
+
+class EmbeddedClient(httpx.AsyncClient):
+ """httpx.AsyncClient subclass that holds a typed reference to the embedded transport."""
+
+ def __init__(self, transport: httpx.AsyncBaseTransport, **kwargs: Any) -> None: # type: ignore[override]
+ super().__init__(transport=transport, **kwargs)
+ self.embedded_transport = transport
+
+
+def _apply_sqlite_overrides() -> None:
+ """Apply SQLite-required settings overrides to whichever Settings object is active.
+
+ Uses a dual-target loop because Settings and SettingsManager.get_settings()
+ may be different objects when cloud/__init__.py was previously imported
+ (e.g., in a mixed test suite where scenario conftest imports cloud).
+ """
+ from skyvern.forge.sdk.settings_manager import SettingsManager # noqa: PLC0415
- return httpx.AsyncClient(transport=EmbeddedServerTransport(), base_url="http://skyvern-embedded")
+ targets = [settings]
+ mgr_settings = SettingsManager.get_settings()
+ if mgr_settings is not settings:
+ targets.append(mgr_settings)
+ for target in targets:
+ target.DATABASE_STRING = "sqlite+aiosqlite:///:memory:"
+ target.DATABASE_REPLICA_STRING = None
+ target.ADDITIONAL_MODULES = []
+ target.OTEL_ENABLED = False
+ if hasattr(target, "ENABLE_CLEANUP_CRON"):
+ target.ENABLE_CLEANUP_CRON = False
diff --git a/skyvern/library/skyvern.py b/skyvern/library/skyvern.py
index 33511ad8d..1c80e6b7a 100644
--- a/skyvern/library/skyvern.py
+++ b/skyvern/library/skyvern.py
@@ -121,6 +121,7 @@ class Skyvern(AsyncSkyvern):
self._environment = environment
self._api_key: str | None = api_key
self._playwright: Playwright | None = None
+ self._embedded_client: httpx.AsyncClient | None = None
@classmethod
def local(
@@ -128,12 +129,10 @@ class Skyvern(AsyncSkyvern):
*,
llm_config: LLMRouterConfig | LLMConfig | None = None,
settings: dict[str, Any] | None = None,
+ use_in_memory_db: bool = False,
) -> "Skyvern":
"""Local/embedded mode: Run Skyvern locally in-process.
- Prerequisites:
- Run `skyvern quickstart` first to set up your local environment and create a .env file
-
Args:
llm_config: Optional custom LLM configuration (LLMConfig or LLMRouterConfig).
If provided, this will be registered as "CUSTOM_LLM" and used as the primary LLM,
@@ -149,71 +148,81 @@ class Skyvern(AsyncSkyvern):
skyvern = Skyvern.local()
```
- Example 2 - Custom LLM with environment variables:
+ Example 2 - Zero-config with in-memory database (no Postgres needed):
```python
from skyvern import Skyvern
- from skyvern.forge.sdk.api.llm.models import LLMConfig
- # Assumes OPENAI_API_KEY is set in your environment
- skyvern = Skyvern.local(
- llm_config=LLMConfig(
- model_name="gpt-4o",
- required_env_vars=["OPENAI_API_KEY"],
- supports_vision=True,
- add_assistant_prefix=False,
- )
- )
+ skyvern = Skyvern.local(use_in_memory_db=True)
```
- Example 3 - Explicitly providing credentials:
+ Example 3 - Custom LLM with environment variables:
```python
from skyvern import Skyvern
- from skyvern.forge.sdk.api.llm.models import LLMConfig, LiteLLMParams
+ from skyvern.forge.sdk.api.llm.models import LLMConfig
+ # Assumes OPENAI_API_KEY is set in your environment
skyvern = Skyvern.local(
llm_config=LLMConfig(
model_name="gpt-4o",
- required_env_vars=[], # No env vars required
+ required_env_vars=["OPENAI_API_KEY"],
supports_vision=True,
add_assistant_prefix=False,
- litellm_params=LiteLLMParams(
- api_base="https://api.openai.com/v1",
- api_key="sk-...", # Your API key
- ),
)
)
```
settings: Optional dictionary of Skyvern settings to override.
These override the corresponding settings from your .env file.
Example: {"MAX_STEPS_PER_RUN": 100, "BROWSER_TYPE": "chromium-headful"}
+ use_in_memory_db: If True, use SQLite in-memory instead of PostgreSQL.
+ No .env file or running Postgres instance required. Defaults to False.
+
+ Zero-config mode supports:
+ - run_task(), extract(), click(), navigate() — full browser automation
+ - Workflow CRUD (create, list, search, get, run)
+ - Artifacts saved to local temp directory (file:// URIs)
+ - Any LLM provider supported by litellm
+
+ Not supported in zero-config mode (requires Skyvern Cloud or Postgres):
+ - Workflow scheduling (requires persistent database)
+ - Cloud browser sessions (requires S3/Azure storage)
+ - Rate limiting (cloud-only)
+ - Multi-worker / multi-process execution (requires Temporal)
+
+ Requirements: LLM API key and Playwright
+ (pip install playwright && playwright install chromium).
Returns:
Skyvern: A Skyvern instance running in local/embedded mode.
"""
from skyvern.library.embedded_server_factory import create_embedded_server # noqa: PLC0415
- if not os.path.exists(".env"):
- raise ValueError("Please run `skyvern quickstart` to set up your local Skyvern environment")
+ if not use_in_memory_db:
+ if not os.path.exists(".env"):
+ raise ValueError("Please run `skyvern quickstart` to set up your local Skyvern environment")
- load_dotenv(".env")
- api_key = os.getenv("SKYVERN_API_KEY")
- if not api_key:
- raise ValueError("SKYVERN_API_KEY is not set. Provide api_key or set SKYVERN_API_KEY in .env file.")
+ load_dotenv(".env")
+ api_key = os.getenv("SKYVERN_API_KEY")
+ if not api_key:
+ raise ValueError("SKYVERN_API_KEY is not set. Provide api_key or set SKYVERN_API_KEY in .env file.")
obj = cls.__new__(cls)
+ embedded_client = create_embedded_server(
+ llm_config=llm_config,
+ settings_overrides=settings,
+ use_in_memory_db=use_in_memory_db,
+ )
+
AsyncSkyvern.__init__(
obj,
environment=SkyvernEnvironment.LOCAL,
- httpx_client=create_embedded_server(
- llm_config=llm_config,
- settings_overrides=settings,
- ),
+ httpx_client=embedded_client,
)
obj._environment = SkyvernEnvironment.LOCAL
obj._api_key = None
obj._playwright = None
+ obj._embedded_client = embedded_client
return obj
@@ -582,9 +591,17 @@ class Skyvern(AsyncSkyvern):
return self._playwright
async def aclose(self) -> None:
- """Close Playwright and release resources."""
+ """Close Playwright, embedded server resources, and release all handles."""
if self._playwright is not None:
try:
await self._playwright.stop()
finally:
self._playwright = None
+
+ if self._embedded_client is not None:
+ from skyvern.library.embedded_server_factory import EmbeddedClient # noqa: PLC0415
+
+ if isinstance(self._embedded_client, EmbeddedClient):
+ await self._embedded_client.embedded_transport.aclose()
+ await self._embedded_client.aclose()
+ self._embedded_client = None
diff --git a/tests/unit/embedded/__init__.py b/tests/unit/embedded/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/unit/embedded/conftest.py b/tests/unit/embedded/conftest.py
new file mode 100644
index 000000000..17f3667ff
--- /dev/null
+++ b/tests/unit/embedded/conftest.py
@@ -0,0 +1,58 @@
+"""Conftest for embedded mode tests.
+
+This conftest intentionally does NOT import from cloud/.
+Embedded tests must work without cloud module side effects.
+
+The _restore_settings fixture is REQUIRED because pytest runs all tests
+in a single process. Without it, _apply_sqlite_overrides() changes
+DATABASE_STRING to SQLite, and subsequent scenario tests (which expect
+Postgres) fail with "no such table: organizations".
+"""
+
+import copy
+
+import pytest
+
+from skyvern.config import settings
+
+
[email protected](autouse=True)
+def _restore_settings():
+ """Save and restore global settings around each embedded test.
+
+ Embedded mode mutates the global settings singleton (DATABASE_STRING,
+ ADDITIONAL_MODULES, OTEL_ENABLED, etc.). Without restoration, subsequent
+ Postgres-based scenario tests in the same pytest process see SQLite
+ settings and fail.
+ """
+ from skyvern.forge.sdk.settings_manager import SettingsManager
+
+ keys_to_save = [
+ "DATABASE_STRING",
+ "DATABASE_REPLICA_STRING",
+ "ADDITIONAL_MODULES",
+ "OTEL_ENABLED",
+ "LLM_KEY",
+ "BROWSER_LOGS_ENABLED",
+ ]
+
+ original_values = {}
+ for key in keys_to_save:
+ val = getattr(settings, key, None)
+ original_values[key] = copy.deepcopy(val) if isinstance(val, (list, dict)) else val
+
+ mgr_originals = {}
+ mgr_settings = SettingsManager.get_settings()
+ if mgr_settings is not settings:
+ for key in keys_to_save:
+ if hasattr(mgr_settings, key):
+ val = getattr(mgr_settings, key)
+ mgr_originals[key] = copy.deepcopy(val) if isinstance(val, (list, dict)) else val
+
+ yield
+
+ for key, val in original_values.items():
+ setattr(settings, key, val)
+ if mgr_settings is not settings:
+ for key, val in mgr_originals.items():
+ setattr(mgr_settings, key, val)
diff --git a/tests/unit/embedded/test_embedded_boundary.py b/tests/unit/embedded/test_embedded_boundary.py
new file mode 100644
index 000000000..b5d235ddf
--- /dev/null
+++ b/tests/unit/embedded/test_embedded_boundary.py
@@ -0,0 +1,140 @@
+"""Regression tests for the embedded mode bootstrap boundary.
+
+These tests verify hermetic isolation between embedded mode and cloud modules.
+They do NOT require an LLM key or Playwright — they test infrastructure only.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+
+def test_no_cloud_in_sys_modules() -> None:
+ """Importing api_app must not trigger cloud/__init__.py in a clean process.
+
+ In a mixed test suite where scenario conftest already imported cloud,
+ this verifies the import itself doesn't RE-trigger cloud. In a clean
+ process (SDK user), this verifies cloud is never loaded.
+ """
+ import importlib
+
+ cloud_before = "cloud" in sys.modules
+ if "skyvern.forge.api_app" in sys.modules:
+ importlib.reload(sys.modules["skyvern.forge.api_app"])
+ else:
+ import skyvern.forge.api_app # noqa: F401
+
+ cloud_after = "cloud" in sys.modules
+ if not cloud_before:
+ assert not cloud_after, (
+ f"cloud was loaded by importing api_app. Modules: {[k for k in sys.modules if k.startswith('cloud')]}"
+ )
+
+
+async def test_bootstrap_creates_db() -> None:
+ """Embedded bootstrap creates SQLite tables, org, and token."""
+ from skyvern import Skyvern
+
+ skyvern = Skyvern.local(use_in_memory_db=True)
+ try:
+ workflows = await skyvern.get_workflows()
+ assert workflows is not None
+ finally:
+ await skyvern.aclose()
+
+
+async def test_artifact_tempdir_is_live() -> None:
+ """StorageFactory points to an existing temp directory after bootstrap."""
+ from skyvern import Skyvern
+ from skyvern.library.embedded_server_factory import EmbeddedClient
+
+ skyvern = Skyvern.local(use_in_memory_db=True)
+ try:
+ await skyvern.get_workflows()
+ client = getattr(skyvern, "_embedded_client", None)
+ assert isinstance(client, EmbeddedClient)
+ artifact_dir = client.embedded_transport._artifact_dir
+ assert artifact_dir is not None
+ assert Path(artifact_dir).exists()
+ assert "skyvern-artifacts-" in artifact_dir
+ finally:
+ await skyvern.aclose()
+
+
+async def test_unique_llm_key_no_collision() -> None:
+ """Sequential clients (create -> close -> create again) use different LLM keys."""
+ from skyvern import Skyvern
+ from skyvern.forge.sdk.api.llm.models import LLMConfig
+
+ config = LLMConfig(
+ model_name="gpt-4o-mini",
+ required_env_vars=[],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ )
+
+ skyvern1 = Skyvern.local(use_in_memory_db=True, llm_config=config)
+ try:
+ await skyvern1.get_workflows()
+ finally:
+ await skyvern1.aclose()
+
+ # Second client should NOT raise DuplicateLLMConfigError
+ skyvern2 = Skyvern.local(use_in_memory_db=True, llm_config=config)
+ try:
+ await skyvern2.get_workflows()
+ finally:
+ await skyvern2.aclose()
+
+
+async def test_second_client_fails_fast() -> None:
+ """Creating a second embedded client WHILE the first is still open raises."""
+ from skyvern import Skyvern
+
+ skyvern1 = Skyvern.local(use_in_memory_db=True)
+ try:
+ with pytest.raises(RuntimeError, match="already active"):
+ Skyvern.local(use_in_memory_db=True)
+ finally:
+ await skyvern1.aclose()
+
+
+async def test_close_without_bootstrap() -> None:
+ """aclose() on a client that never made a request is a no-op."""
+ from skyvern import Skyvern
+
+ skyvern = Skyvern.local(use_in_memory_db=True)
+ await skyvern.aclose()
+
+
+async def test_double_close_idempotent() -> None:
+ """Calling aclose() twice does not raise."""
+ from skyvern import Skyvern
+
+ skyvern = Skyvern.local(use_in_memory_db=True)
+ try:
+ await skyvern.get_workflows()
+ finally:
+ await skyvern.aclose()
+ await skyvern.aclose()
+
+
+async def test_blocked_settings_rejected() -> None:
+ """Attempting to override OTEL_ENABLED or ENABLE_CLEANUP_CRON raises ValueError."""
+ from skyvern import Skyvern
+
+ skyvern = Skyvern.local(use_in_memory_db=True, settings={"OTEL_ENABLED": True})
+ try:
+ # Validation happens on first request (lazy bootstrap)
+ with pytest.raises(ValueError, match="Cannot override"):
+ await skyvern.get_workflows()
+ finally:
+ await skyvern.aclose()
diff --git a/tests/unit/embedded/test_embedded_e2e.py b/tests/unit/embedded/test_embedded_e2e.py
new file mode 100644
index 000000000..44fd79f9b
--- /dev/null
+++ b/tests/unit/embedded/test_embedded_e2e.py
@@ -0,0 +1,130 @@
+"""End-to-end acceptance tests for Skyvern.local(use_in_memory_db=True).
+
+Exercises the full embedded mode path:
+- SQLite in-memory database
+- BackgroundTaskExecutor (default in-process executor)
+- LocalStorage with temp directory (rebuilt during bootstrap)
+- Lifecycle cleanup (engine disposal, client close)
+
+Requires OPENAI_API_KEY and Playwright browsers installed.
+Skipped in CI unless the key is available.
+"""
+
+import os
+from pathlib import Path
+
+import pytest
+
+from skyvern.forge.sdk.api.llm.models import LLMConfig
+
+
+def _has_playwright_browser() -> bool:
+ """Check that Playwright's chromium binary exists for the current installed version."""
+ try:
+ from playwright.sync_api import sync_playwright # noqa: PLC0415
+
+ with sync_playwright() as p:
+ return Path(p.chromium.executable_path).exists()
+ except Exception:
+ return False
+
+
+_skip_no_llm_or_browser = pytest.mark.skipif(
+ not os.environ.get("OPENAI_API_KEY") or not _has_playwright_browser(),
+ reason="Requires OPENAI_API_KEY and Playwright browsers installed (run: playwright install chromium)",
+)
+
+
+@_skip_no_llm_or_browser
+async def test_embedded_mode_run_task() -> None:
+ """Path 1: run_task() exercises BackgroundTaskExecutor + polling + full DB lifecycle.
+
+ Flow: httpx -> ASGITransport -> FastAPI -> BackgroundTaskExecutor ->
+ ForgeAgent.execute_step() -> LLM + browser -> artifact save -> task complete
+ """
+ from skyvern import Skyvern
+
+ skyvern = Skyvern.local(
+ use_in_memory_db=True,
+ llm_config=LLMConfig(
+ model_name="gpt-4o-mini",
+ required_env_vars=["OPENAI_API_KEY"],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ ),
+ )
+
+ try:
+ result = await skyvern.run_task(
+ prompt="Extract the page title",
+ url="https://example.com",
+ wait_for_completion=True,
+ timeout=120,
+ )
+ assert result is not None
+ assert result.status is not None
+ finally:
+ await skyvern.aclose()
+
+
+@_skip_no_llm_or_browser
+async def test_embedded_mode_page_extract() -> None:
+ """Path 2: page.extract() exercises direct in-process LLM + browser path.
+
+ Flow: page.extract() -> run_sdk_action() -> LLM call -> result
+ Does NOT go through BackgroundTaskExecutor or polling.
+ """
+ from skyvern import Skyvern
+
+ skyvern = Skyvern.local(
+ use_in_memory_db=True,
+ llm_config=LLMConfig(
+ model_name="gpt-4o-mini",
+ required_env_vars=["OPENAI_API_KEY"],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ ),
+ )
+
+ try:
+ browser = await skyvern.launch_local_browser()
+ page = await browser.get_working_page()
+ await page.goto("https://example.com")
+ result = await page.extract("What is the title of this page?")
+ assert result is not None
+ finally:
+ await skyvern.aclose()
+
+
+async def test_embedded_mode_bootstrap_and_artifacts() -> None:
+ """Verify embedded bootstrap creates SQLite DB, org, token, and redirects artifacts to tempdir.
+
+ Does NOT require an LLM key — tests the infrastructure, not the LLM path.
+ Bootstrap is lazy (happens on first request), so we trigger it via get_workflows().
+ """
+ from skyvern import Skyvern
+
+ skyvern = Skyvern.local(use_in_memory_db=True)
+
+ try:
+ workflows = await skyvern.get_workflows()
+ assert workflows is not None
+
+ embedded_client = getattr(skyvern, "_embedded_client", None)
+ assert embedded_client is not None
+
+ from skyvern.library.embedded_server_factory import EmbeddedClient
+
+ assert isinstance(embedded_client, EmbeddedClient)
+ transport = embedded_client.embedded_transport
+ assert transport is not None
+
+ artifact_dir = getattr(transport, "_artifact_dir", None)
+ assert artifact_dir is not None
+ assert Path(artifact_dir).exists()
+ assert "skyvern-artifacts-" in artifact_dir
+ finally:
+ await skyvern.aclose()
diff --git a/uv.lock b/uv.lock
index e32b5de2e..7c72fed05 100644
--- a/uv.lock
+++ b/uv.lock
@@ -5487,6 +5487,7 @@ dependencies = [
{ name = "aiofiles" },
{ name = "aiohttp" },
{ name = "aioredlock" },
+ { name = "aiosqlite" },
{ name = "alembic" },
{ name = "anthropic" },
{ name = "asyncache" },
@@ -5608,6 +5609,7 @@ requires-dist = [
{ name = "aiofiles", specifier = ">=24.1.0,<25" },
{ name = "aiohttp", specifier = ">=3.8.5,<4" },
{ name = "aioredlock", specifier = ">=0.7.3,<0.8" },
+ { name = "aiosqlite", specifier = ">=0.21.0,<0.22" },
{ name = "alembic", specifier = ">=1.12.1,<2" },
{ name = "anthropic", specifier = ">=0.50.0,<0.51" },
{ name = "asyncache", specifier = ">=0.3.1,<0.4" },