Skip to content

Automate browser based workflows with AI

Give both concurrent-index migration tests a scratch database so an open transaction cannot stall the build (SKY-18402) (#8926)

Shuchang Zhenge66c233d0c41
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index 81ba476cb..de111663e 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -2128,7 +2128,11 @@ class WorkflowParametersRepository(BaseRepository):
query = (
select(WorkflowCopilotChatMessageModel)
.filter(WorkflowCopilotChatMessageModel.workflow_copilot_chat_id == workflow_copilot_chat_id)
- .order_by(WorkflowCopilotChatMessageModel.workflow_copilot_chat_message_id.asc())
+ # Within one second, ids follow the worker hash and a wrapping 10-bit sequence, not write order.
+ .order_by(
+ WorkflowCopilotChatMessageModel.created_at.asc(),
+ WorkflowCopilotChatMessageModel.workflow_copilot_chat_message_id.asc(),
+ )
)
messages = (await session.scalars(query)).all()
return [convert_to_workflow_copilot_chat_message(message, self.debug_enabled) for message in messages]
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
index 218902c54..cf1a77305 100644
--- a/tests/unit/conftest.py
+++ b/tests/unit/conftest.py
@@ -10,6 +10,7 @@ import os
import shutil
import sys
import threading
+import uuid
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iterator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import dataclass, field
@@ -28,10 +29,12 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from playwright.async_api import Download
from playwright.async_api import Error as PlaywrightError
-from sqlalchemy import create_engine, func, select
+from sqlalchemy import Engine, create_engine, func, select
+from sqlalchemy.engine import URL, make_url
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
import skyvern._cli_bootstrap as cli_bootstrap
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.prompts import prompt_engine
@@ -68,6 +71,26 @@ from tests.unit.litellm_model_registry import registered_gpt56_litellm_models
# Four distinct ways to leave the legacy downloads root; each defeats a different weak check.
LEGACY_DOWNLOAD_ESCAPE_CASES = ("parent_traversal", "encoded_dot_dot", "sibling_prefix", "symlink_escape")
+LOOPBACK_POSTGRES_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
+# The schema fixture only adds and drops a schema, so it also accepts the compose service name.
+LOCAL_POSTGRES_HOSTS = LOOPBACK_POSTGRES_HOSTS | {"postgres"}
+# Any other query key (host, hostaddr, service, dbname, port, user, options, ...) can change which server or database
+# libpq reaches, so unknown keys are rejected instead of listing the bad ones.
+POSTGRES_NON_ROUTING_QUERY_KEYS = frozenset({"connect_timeout", "application_name", "sslmode"})
+# These env vars redirect a connection even when the URL names its host; PGPORT does so only when the URL has no port.
+POSTGRES_HOST_OVERRIDE_ENV_VARS = ("PGHOSTADDR", "PGSERVICE")
+
+
+def skip_unless_local_postgres(url: URL, hosts: frozenset[str] = LOCAL_POSTGRES_HOSTS) -> None:
+ if url.get_backend_name() != "postgresql" or url.host not in hosts:
+ pytest.skip(f"needs a PostgreSQL on one of: {', '.join(sorted(hosts))}")
+ overrides = sorted(set(url.query) - POSTGRES_NON_ROUTING_QUERY_KEYS)
+ overrides += [name for name in POSTGRES_HOST_OVERRIDE_ENV_VARS if os.environ.get(name)]
+ if url.port is None and os.environ.get("PGPORT"):
+ overrides.append("PGPORT")
+ if overrides:
+ pytest.skip(f"{', '.join(overrides)} could redirect the connection away from the intended local PostgreSQL")
+
def fake_download_registration_app(
register: Callable[..., Awaitable[None]], rows: list[Artifact], storage: BaseStorage | None = None
@@ -415,6 +438,34 @@ async def sqlite_engine(sqlite_engine_factory: Callable[[], AsyncEngine]) -> Asy
return sqlite_engine_factory()
+def postgres_scratch_database() -> Iterator[Engine]:
+ url = make_url(str(settings.DATABASE_STRING))
+ # Loopback only: a generic name like postgres could resolve to a shared server, and this drops a database.
+ skip_unless_local_postgres(url, LOOPBACK_POSTGRES_HOSTS)
+ url = url.set(drivername="postgresql+psycopg")
+ # A database, not a schema: CREATE INDEX CONCURRENTLY waits for every older snapshot in its database, so a
+ # transaction another test leaves open in the shared one would stall the build until statement_timeout.
+ database = f"test_scratch_{uuid.uuid4().hex}"
+ admin = create_engine(url, isolation_level="AUTOCOMMIT")
+ try:
+ with admin.connect() as connection:
+ if not connection.exec_driver_sql(
+ "SELECT rolcreatedb OR rolsuper FROM pg_roles WHERE rolname = current_user"
+ ).scalar_one():
+ pytest.skip("needs a PostgreSQL role that can create databases")
+ connection.exec_driver_sql(f'CREATE DATABASE "{database}"')
+ engine = create_engine(url.set(database=database))
+ try:
+ yield engine
+ finally:
+ engine.dispose()
+ with admin.connect() as connection:
+ connection.exec_driver_sql(f'DROP DATABASE "{database}" WITH (FORCE)')
+ finally:
+ admin.dispose()
+
+
def make_input_element_mock(*, element_id: str = "AADC", attrs: dict[str, object] | None = None) -> MagicMock:
# SkyvernElement double for handle_input_text_action tests. attrs=None makes every get_attr return
# None (plain search-bar case); pass a dict to drive specific attrs (e.g. a combobox's role /
diff --git a/tests/unit/forge/sdk/db/conftest.py b/tests/unit/forge/sdk/db/conftest.py
index f75d879bd..5e72a8a93 100644
--- a/tests/unit/forge/sdk/db/conftest.py
+++ b/tests/unit/forge/sdk/db/conftest.py
@@ -17,8 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from skyvern.config import settings
from skyvern.forge.sdk.db.agent_db import AgentDB, _build_engine
from skyvern.forge.sdk.db.models import Base
-
-_LOCAL_POSTGRES_HOSTS = {"localhost", "127.0.0.1", "::1", "postgres"}
+from tests.unit.conftest import skip_unless_local_postgres
@pytest_asyncio.fixture
@@ -39,8 +38,7 @@ async def agent_db(db_engine: Any) -> AsyncGenerator[AgentDB]:
def postgres_scratch_schema_url() -> Iterator[URL]:
"""DATABASE_STRING aimed at a throwaway schema built from the models, so no test row lands in public."""
url = make_url(settings.DATABASE_STRING)
- if url.get_backend_name() != "postgresql" or url.host not in _LOCAL_POSTGRES_HOSTS:
- pytest.skip("needs a local or CI PostgreSQL")
+ skip_unless_local_postgres(url)
schema = f"test_scratch_{uuid.uuid4().hex}"
admin = create_engine(url.set(drivername="postgresql+psycopg"))
try:
diff --git a/tests/unit/test_debug_sessions_lookup_index_migration.py b/tests/unit/test_debug_sessions_lookup_index_migration.py
index 5278264d6..7c2a4006b 100644
--- a/tests/unit/test_debug_sessions_lookup_index_migration.py
+++ b/tests/unit/test_debug_sessions_lookup_index_migration.py
@@ -1,23 +1,19 @@
import functools
import importlib.util
import re
-import uuid
-from collections.abc import Iterator
from datetime import datetime, timedelta
from pathlib import Path
from types import ModuleType
-from typing import Any, NamedTuple
+from typing import Any
import pytest
-from sqlalchemy import Engine, create_engine, event, insert, text
-from sqlalchemy.engine import make_url
+from sqlalchemy import Engine, event, insert, text
from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.schema import CreateTable
from alembic.operations import Operations
from alembic.runtime.migration import MigrationContext
-from skyvern.config import settings
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.models import DebugSessionModel
@@ -39,28 +35,11 @@ def _migration() -> ModuleType:
return module
-class _Schema(NamedTuple):
- engine: Engine
- connect_args: dict[str, str]
-
-
@pytest.fixture
-def scratch_schema() -> Iterator[_Schema]:
- url = make_url(str(settings.DATABASE_STRING))
- if url.get_backend_name() != "postgresql":
- pytest.skip("requires PostgreSQL")
- schema = f"debug_sessions_index_{uuid.uuid4().hex}"
- connect_args = {"options": f"-csearch_path={schema}"}
- engine = create_engine(url.set(drivername="postgresql+psycopg"), connect_args=connect_args)
- with engine.begin() as connection:
- connection.exec_driver_sql(f'CREATE SCHEMA "{schema}"')
+def scratch_database(postgres_scratch_database: Engine) -> Engine:
+ with postgres_scratch_database.begin() as connection:
connection.execute(CreateTable(DebugSessionModel.__table__))
- try:
- yield _Schema(engine, connect_args)
- finally:
- with engine.begin() as connection:
- connection.exec_driver_sql(f'DROP SCHEMA "{schema}" CASCADE')
- engine.dispose()
+ return postgres_scratch_database
def _upgrade(engine: Engine) -> None:
@@ -120,13 +99,13 @@ def _generic_plan(engine: Engine, statement: str, parameters: dict[str, Any]) ->
@pytest.mark.asyncio
async def test_lookup_returns_the_newest_open_session_through_the_index_under_a_generic_plan(
- scratch_schema: _Schema,
+ scratch_database: Engine,
) -> None:
- schema_engine = scratch_schema.engine
- _upgrade(schema_engine)
+ engine = scratch_database
+ _upgrade(engine)
organization_id = "o_debug_lookup"
started = datetime(2026, 9, 1)
- with schema_engine.begin() as connection:
+ with engine.begin() as connection:
connection.execute(
insert(DebugSessionModel),
[
@@ -147,7 +126,7 @@ async def test_lookup_returns_the_newest_open_session_through_the_index_under_a_
],
)
- async_engine = create_async_engine(schema_engine.url, connect_args=scratch_schema.connect_args)
+ async_engine = create_async_engine(engine.url)
statements: list[tuple[str, dict[str, Any]]] = []
def capture(_conn: Any, _cursor: Any, statement: str, parameters: Any, *_args: Any) -> None:
@@ -169,26 +148,26 @@ async def test_lookup_returns_the_newest_open_session_through_the_index_under_a_
[(statement, parameters)] = statements
# psycopg prepares a statement after five runs on a connection, and Postgres may then plan it generically,
# with status as an unknown parameter.
- plan = _generic_plan(schema_engine, statement, parameters)
+ plan = _generic_plan(engine, statement, parameters)
assert f"Index Scan Backward using {INDEX_NAME}" in plan, plan
assert "Sort" not in plan, plan
-def test_upgrade_rebuilds_an_invalid_index_left_by_a_failed_concurrent_build(scratch_schema: _Schema) -> None:
- schema_engine = scratch_schema.engine
- with schema_engine.begin() as connection:
+def test_upgrade_rebuilds_an_invalid_index_left_by_a_failed_concurrent_build(scratch_database: Engine) -> None:
+ engine = scratch_database
+ with engine.begin() as connection:
connection.execute(insert(DebugSessionModel), [_session_row("o_debug_lookup", "only")])
- with schema_engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
+ with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
with pytest.raises(DBAPIError, match="division by zero"):
connection.exec_driver_sql(
f"CREATE INDEX CONCURRENTLY {INDEX_NAME} ON debug_sessions ((length(user_id) / 0))"
)
- leftover = _index_state(schema_engine)
+ leftover = _index_state(engine)
assert leftover is not None and leftover[0] is False
- _upgrade(schema_engine)
+ _upgrade(engine)
- state = _index_state(schema_engine)
+ state = _index_state(engine)
assert state is not None
valid, definition = state
assert valid is True
diff --git a/tests/unit/test_reuse_browser_session_migration.py b/tests/unit/test_reuse_browser_session_migration.py
index 4c4300df7..48af34345 100644
--- a/tests/unit/test_reuse_browser_session_migration.py
+++ b/tests/unit/test_reuse_browser_session_migration.py
@@ -1,15 +1,13 @@
import importlib.util
-import uuid
from contextlib import contextmanager
from pathlib import Path
from types import SimpleNamespace
from typing import Iterator
import pytest
-from sqlalchemy import create_engine
-from sqlalchemy.engine import Connection, make_url
+from sqlalchemy import Engine
+from sqlalchemy.engine import Connection
-from skyvern.config import settings
from skyvern.forge.sdk.workflow.sequential_key import resolve_reuse_bound_key
@@ -116,84 +114,70 @@ class _ConnectionOp:
def test_binding_migration_replaces_invalid_concurrent_index_on_postgres(
- monkeypatch: pytest.MonkeyPatch,
+ monkeypatch: pytest.MonkeyPatch, postgres_scratch_database: Engine
) -> None:
- database_url = make_url(str(settings.DATABASE_STRING))
- if database_url.get_backend_name() != "postgresql":
- pytest.skip("requires PostgreSQL")
- database_url = database_url.set(drivername="postgresql+psycopg")
- engine = create_engine(database_url, isolation_level="AUTOCOMMIT")
- schema = f"reuse_migration_{uuid.uuid4().hex}"
+ engine = postgres_scratch_database.execution_options(isolation_level="AUTOCOMMIT")
migration = _load_migration_module()
- try:
- with engine.connect() as connection:
- connection.exec_driver_sql(f'CREATE SCHEMA "{schema}"')
- connection.exec_driver_sql(f'SET search_path TO "{schema}"')
- connection.exec_driver_sql(
- "CREATE TABLE persistent_browser_sessions ("
- "id INTEGER PRIMARY KEY, "
- "organization_id VARCHAR NOT NULL, "
- "bound_workflow_permanent_id VARCHAR, "
- "bound_key VARCHAR, "
- "deleted_at TIMESTAMP, "
- "status VARCHAR NOT NULL)"
- )
+ with engine.connect() as connection:
+ connection.exec_driver_sql(
+ "CREATE TABLE persistent_browser_sessions ("
+ "id INTEGER PRIMARY KEY, "
+ "organization_id VARCHAR NOT NULL, "
+ "bound_workflow_permanent_id VARCHAR, "
+ "bound_key VARCHAR, "
+ "deleted_at TIMESTAMP, "
+ "status VARCHAR NOT NULL)"
+ )
+ connection.exec_driver_sql(
+ "INSERT INTO persistent_browser_sessions "
+ "(id, organization_id, bound_workflow_permanent_id, bound_key, status) VALUES "
+ "(1, 'org', 'wpid', NULL, 'running'), "
+ "(2, 'org', 'wpid', NULL, 'running')"
+ )
+ with pytest.raises(Exception):
connection.exec_driver_sql(
- "INSERT INTO persistent_browser_sessions "
- "(id, organization_id, bound_workflow_permanent_id, bound_key, status) VALUES "
- "(1, 'org', 'wpid', NULL, 'running'), "
- "(2, 'org', 'wpid', NULL, 'running')"
- )
- with pytest.raises(Exception):
- connection.exec_driver_sql(
- "CREATE UNIQUE INDEX CONCURRENTLY uq_pbs_live_workflow_binding "
- "ON persistent_browser_sessions "
- "(organization_id, bound_workflow_permanent_id, COALESCE(bound_key, '')) "
- "WHERE bound_workflow_permanent_id IS NOT NULL "
- "AND deleted_at IS NULL "
- "AND status IN ('created', 'running', 'retry')"
- )
- connection.rollback()
- invalid = connection.exec_driver_sql(
- "SELECT indisvalid FROM pg_catalog.pg_index "
- "WHERE indexrelid = to_regclass('uq_pbs_live_workflow_binding')"
- ).scalar_one()
- assert invalid is False
- connection.exec_driver_sql("DELETE FROM persistent_browser_sessions WHERE id = 2")
-
- monkeypatch.setattr(migration, "op", _ConnectionOp(connection))
- migration.upgrade()
-
- valid, unique = connection.exec_driver_sql(
- "SELECT indisvalid, indisunique FROM pg_catalog.pg_index "
- "WHERE indexrelid = to_regclass('uq_pbs_live_workflow_binding')"
- ).one()
- assert valid is True
- assert unique is True
- oversized_key, _ = resolve_reuse_bound_key(
- SimpleNamespace(
- workflow_definition=SimpleNamespace(blocks=[]),
- browser_profile_key="{{ payload }}",
- sequential_key=None,
- workflow_permanent_id="wpid",
- ),
- {"payload": "x" * 4096},
- {},
- )
- assert oversized_key.startswith("profile:sha256:")
- connection.exec_driver_sql(
- "INSERT INTO persistent_browser_sessions "
- "(id, organization_id, bound_workflow_permanent_id, bound_key, status) "
- "VALUES (3, 'org', 'wpid', %s, 'running')",
- (oversized_key,),
- )
- assert (
- connection.exec_driver_sql(
- "SELECT bound_key FROM persistent_browser_sessions WHERE id = 3"
- ).scalar_one()
- == oversized_key
+ "CREATE UNIQUE INDEX CONCURRENTLY uq_pbs_live_workflow_binding "
+ "ON persistent_browser_sessions "
+ "(organization_id, bound_workflow_permanent_id, COALESCE(bound_key, '')) "
+ "WHERE bound_workflow_permanent_id IS NOT NULL "
+ "AND deleted_at IS NULL "
+ "AND status IN ('created', 'running', 'retry')"
)
- connection.exec_driver_sql(f'DROP SCHEMA "{schema}" CASCADE')
- finally:
- engine.dispose()
+ connection.rollback()
+ invalid = connection.exec_driver_sql(
+ "SELECT indisvalid FROM pg_catalog.pg_index WHERE indexrelid = to_regclass('uq_pbs_live_workflow_binding')"
+ ).scalar_one()
+ assert invalid is False
+ connection.exec_driver_sql("DELETE FROM persistent_browser_sessions WHERE id = 2")
+
+ monkeypatch.setattr(migration, "op", _ConnectionOp(connection))
+ migration.upgrade()
+
+ valid, unique = connection.exec_driver_sql(
+ "SELECT indisvalid, indisunique FROM pg_catalog.pg_index "
+ "WHERE indexrelid = to_regclass('uq_pbs_live_workflow_binding')"
+ ).one()
+ assert valid is True
+ assert unique is True
+ oversized_key, _ = resolve_reuse_bound_key(
+ SimpleNamespace(
+ workflow_definition=SimpleNamespace(blocks=[]),
+ browser_profile_key="{{ payload }}",
+ sequential_key=None,
+ workflow_permanent_id="wpid",
+ ),
+ {"payload": "x" * 4096},
+ {},
+ )
+ assert oversized_key.startswith("profile:sha256:")
+ connection.exec_driver_sql(
+ "INSERT INTO persistent_browser_sessions "
+ "(id, organization_id, bound_workflow_permanent_id, bound_key, status) "
+ "VALUES (3, 'org', 'wpid', %s, 'running')",
+ (oversized_key,),
+ )
+ assert (
+ connection.exec_driver_sql("SELECT bound_key FROM persistent_browser_sessions WHERE id = 3").scalar_one()
+ == oversized_key
+ )