Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Give both concurrent-index migration tests a scratch database so an open transaction cannot stall the build (SKY-18402) (#8926)
Shuchang Zhenge66c233d0c41
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.pyindex 81ba476cb..de111663e 100644--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py@@ -2128,7 +2128,11 @@ class WorkflowParametersRepository(BaseRepository):query = (select(WorkflowCopilotChatMessageModel).filter(WorkflowCopilotChatMessageModel.workflow_copilot_chat_id == workflow_copilot_chat_id)- .order_by(WorkflowCopilotChatMessageModel.workflow_copilot_chat_message_id.asc())+ # Within one second, ids follow the worker hash and a wrapping 10-bit sequence, not write order.+ .order_by(+ WorkflowCopilotChatMessageModel.created_at.asc(),+ WorkflowCopilotChatMessageModel.workflow_copilot_chat_message_id.asc(),+ ))messages = (await session.scalars(query)).all()return [convert_to_workflow_copilot_chat_message(message, self.debug_enabled) for message in messages]diff --git a/tests/unit/conftest.py b/tests/unit/conftest.pyindex 218902c54..cf1a77305 100644--- a/tests/unit/conftest.py+++ b/tests/unit/conftest.py@@ -10,6 +10,7 @@ import osimport shutilimport sysimport threading+import uuidfrom collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Iteratorfrom contextlib import AbstractAsyncContextManager, asynccontextmanagerfrom dataclasses import dataclass, field@@ -28,10 +29,12 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessorfrom opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporterfrom playwright.async_api import Downloadfrom playwright.async_api import Error as PlaywrightError-from sqlalchemy import create_engine, func, select+from sqlalchemy import Engine, create_engine, func, select+from sqlalchemy.engine import URL, make_urlfrom sqlalchemy.ext.asyncio import AsyncEngine, create_async_engineimport skyvern._cli_bootstrap as cli_bootstrap+from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.agent_functions import AgentFunctionfrom skyvern.forge.prompts import prompt_engine@@ -68,6 +71,26 @@ from tests.unit.litellm_model_registry import registered_gpt56_litellm_models# Four distinct ways to leave the legacy downloads root; each defeats a different weak check.LEGACY_DOWNLOAD_ESCAPE_CASES = ("parent_traversal", "encoded_dot_dot", "sibling_prefix", "symlink_escape")+LOOPBACK_POSTGRES_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})+# The schema fixture only adds and drops a schema, so it also accepts the compose service name.+LOCAL_POSTGRES_HOSTS = LOOPBACK_POSTGRES_HOSTS | {"postgres"}+# Any other query key (host, hostaddr, service, dbname, port, user, options, ...) can change which server or database+# libpq reaches, so unknown keys are rejected instead of listing the bad ones.+POSTGRES_NON_ROUTING_QUERY_KEYS = frozenset({"connect_timeout", "application_name", "sslmode"})+# These env vars redirect a connection even when the URL names its host; PGPORT does so only when the URL has no port.+POSTGRES_HOST_OVERRIDE_ENV_VARS = ("PGHOSTADDR", "PGSERVICE")+++def skip_unless_local_postgres(url: URL, hosts: frozenset[str] = LOCAL_POSTGRES_HOSTS) -> None:+ if url.get_backend_name() != "postgresql" or url.host not in hosts:+ pytest.skip(f"needs a PostgreSQL on one of: {', '.join(sorted(hosts))}")+ overrides = sorted(set(url.query) - POSTGRES_NON_ROUTING_QUERY_KEYS)+ overrides += [name for name in POSTGRES_HOST_OVERRIDE_ENV_VARS if os.environ.get(name)]+ if url.port is None and os.environ.get("PGPORT"):+ overrides.append("PGPORT")+ if overrides:+ pytest.skip(f"{', '.join(overrides)} could redirect the connection away from the intended local PostgreSQL")+def fake_download_registration_app(register: Callable[..., Awaitable[None]], rows: list[Artifact], storage: BaseStorage | None = None@@ -415,6 +438,34 @@ async def sqlite_engine(sqlite_engine_factory: Callable[[], AsyncEngine]) -> Asyreturn sqlite_engine_factory()+def postgres_scratch_database() -> Iterator[Engine]:+ url = make_url(str(settings.DATABASE_STRING))+ # Loopback only: a generic name like postgres could resolve to a shared server, and this drops a database.+ skip_unless_local_postgres(url, LOOPBACK_POSTGRES_HOSTS)+ url = url.set(drivername="postgresql+psycopg")+ # A database, not a schema: CREATE INDEX CONCURRENTLY waits for every older snapshot in its database, so a+ # transaction another test leaves open in the shared one would stall the build until statement_timeout.+ database = f"test_scratch_{uuid.uuid4().hex}"+ admin = create_engine(url, isolation_level="AUTOCOMMIT")+ try:+ with admin.connect() as connection:+ if not connection.exec_driver_sql(+ "SELECT rolcreatedb OR rolsuper FROM pg_roles WHERE rolname = current_user"+ ).scalar_one():+ pytest.skip("needs a PostgreSQL role that can create databases")+ connection.exec_driver_sql(f'CREATE DATABASE "{database}"')+ engine = create_engine(url.set(database=database))+ try:+ yield engine+ finally:+ engine.dispose()+ with admin.connect() as connection:+ connection.exec_driver_sql(f'DROP DATABASE "{database}" WITH (FORCE)')+ finally:+ admin.dispose()++def make_input_element_mock(*, element_id: str = "AADC", attrs: dict[str, object] | None = None) -> MagicMock:# SkyvernElement double for handle_input_text_action tests. attrs=None makes every get_attr return# None (plain search-bar case); pass a dict to drive specific attrs (e.g. a combobox's role /diff --git a/tests/unit/forge/sdk/db/conftest.py b/tests/unit/forge/sdk/db/conftest.pyindex f75d879bd..5e72a8a93 100644--- a/tests/unit/forge/sdk/db/conftest.py+++ b/tests/unit/forge/sdk/db/conftest.py@@ -17,8 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine, create_async_enginefrom skyvern.config import settingsfrom skyvern.forge.sdk.db.agent_db import AgentDB, _build_enginefrom skyvern.forge.sdk.db.models import Base--_LOCAL_POSTGRES_HOSTS = {"localhost", "127.0.0.1", "::1", "postgres"}+from tests.unit.conftest import skip_unless_local_postgres@pytest_asyncio.fixture@@ -39,8 +38,7 @@ async def agent_db(db_engine: Any) -> AsyncGenerator[AgentDB]:def postgres_scratch_schema_url() -> Iterator[URL]:"""DATABASE_STRING aimed at a throwaway schema built from the models, so no test row lands in public."""url = make_url(settings.DATABASE_STRING)- if url.get_backend_name() != "postgresql" or url.host not in _LOCAL_POSTGRES_HOSTS:- pytest.skip("needs a local or CI PostgreSQL")+ skip_unless_local_postgres(url)schema = f"test_scratch_{uuid.uuid4().hex}"admin = create_engine(url.set(drivername="postgresql+psycopg"))try:diff --git a/tests/unit/test_debug_sessions_lookup_index_migration.py b/tests/unit/test_debug_sessions_lookup_index_migration.pyindex 5278264d6..7c2a4006b 100644--- a/tests/unit/test_debug_sessions_lookup_index_migration.py+++ b/tests/unit/test_debug_sessions_lookup_index_migration.py@@ -1,23 +1,19 @@import functoolsimport importlib.utilimport re-import uuid-from collections.abc import Iteratorfrom datetime import datetime, timedeltafrom pathlib import Pathfrom types import ModuleType-from typing import Any, NamedTuple+from typing import Anyimport pytest-from sqlalchemy import Engine, create_engine, event, insert, text-from sqlalchemy.engine import make_url+from sqlalchemy import Engine, event, insert, textfrom sqlalchemy.exc import DBAPIErrorfrom sqlalchemy.ext.asyncio import create_async_enginefrom sqlalchemy.schema import CreateTablefrom alembic.operations import Operationsfrom alembic.runtime.migration import MigrationContext-from skyvern.config import settingsfrom skyvern.forge.sdk.db.agent_db import AgentDBfrom skyvern.forge.sdk.db.models import DebugSessionModel@@ -39,28 +35,11 @@ def _migration() -> ModuleType:return module-class _Schema(NamedTuple):- engine: Engine- connect_args: dict[str, str]--@pytest.fixture-def scratch_schema() -> Iterator[_Schema]:- url = make_url(str(settings.DATABASE_STRING))- if url.get_backend_name() != "postgresql":- pytest.skip("requires PostgreSQL")- schema = f"debug_sessions_index_{uuid.uuid4().hex}"- connect_args = {"options": f"-csearch_path={schema}"}- engine = create_engine(url.set(drivername="postgresql+psycopg"), connect_args=connect_args)- with engine.begin() as connection:- connection.exec_driver_sql(f'CREATE SCHEMA "{schema}"')+def scratch_database(postgres_scratch_database: Engine) -> Engine:+ with postgres_scratch_database.begin() as connection:connection.execute(CreateTable(DebugSessionModel.__table__))- try:- yield _Schema(engine, connect_args)- finally:- with engine.begin() as connection:- connection.exec_driver_sql(f'DROP SCHEMA "{schema}" CASCADE')- engine.dispose()+ return postgres_scratch_databasedef _upgrade(engine: Engine) -> None:@@ -120,13 +99,13 @@ def _generic_plan(engine: Engine, statement: str, parameters: dict[str, Any]) ->@pytest.mark.asyncioasync def test_lookup_returns_the_newest_open_session_through_the_index_under_a_generic_plan(- scratch_schema: _Schema,+ scratch_database: Engine,) -> None:- schema_engine = scratch_schema.engine- _upgrade(schema_engine)+ engine = scratch_database+ _upgrade(engine)organization_id = "o_debug_lookup"started = datetime(2026, 9, 1)- with schema_engine.begin() as connection:+ with engine.begin() as connection:connection.execute(insert(DebugSessionModel),[@@ -147,7 +126,7 @@ async def test_lookup_returns_the_newest_open_session_through_the_index_under_a_],)- async_engine = create_async_engine(schema_engine.url, connect_args=scratch_schema.connect_args)+ async_engine = create_async_engine(engine.url)statements: list[tuple[str, dict[str, Any]]] = []def capture(_conn: Any, _cursor: Any, statement: str, parameters: Any, *_args: Any) -> None:@@ -169,26 +148,26 @@ async def test_lookup_returns_the_newest_open_session_through_the_index_under_a_[(statement, parameters)] = statements# psycopg prepares a statement after five runs on a connection, and Postgres may then plan it generically,# with status as an unknown parameter.- plan = _generic_plan(schema_engine, statement, parameters)+ plan = _generic_plan(engine, statement, parameters)assert f"Index Scan Backward using {INDEX_NAME}" in plan, planassert "Sort" not in plan, plan-def test_upgrade_rebuilds_an_invalid_index_left_by_a_failed_concurrent_build(scratch_schema: _Schema) -> None:- schema_engine = scratch_schema.engine- with schema_engine.begin() as connection:+def test_upgrade_rebuilds_an_invalid_index_left_by_a_failed_concurrent_build(scratch_database: Engine) -> None:+ engine = scratch_database+ with engine.begin() as connection:connection.execute(insert(DebugSessionModel), [_session_row("o_debug_lookup", "only")])- with schema_engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:+ with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:with pytest.raises(DBAPIError, match="division by zero"):connection.exec_driver_sql(f"CREATE INDEX CONCURRENTLY {INDEX_NAME} ON debug_sessions ((length(user_id) / 0))")- leftover = _index_state(schema_engine)+ leftover = _index_state(engine)assert leftover is not None and leftover[0] is False- _upgrade(schema_engine)+ _upgrade(engine)- state = _index_state(schema_engine)+ state = _index_state(engine)assert state is not Nonevalid, definition = stateassert valid is Truediff --git a/tests/unit/test_reuse_browser_session_migration.py b/tests/unit/test_reuse_browser_session_migration.pyindex 4c4300df7..48af34345 100644--- a/tests/unit/test_reuse_browser_session_migration.py+++ b/tests/unit/test_reuse_browser_session_migration.py@@ -1,15 +1,13 @@import importlib.util-import uuidfrom contextlib import contextmanagerfrom pathlib import Pathfrom types import SimpleNamespacefrom typing import Iteratorimport pytest-from sqlalchemy import create_engine-from sqlalchemy.engine import Connection, make_url+from sqlalchemy import Engine+from sqlalchemy.engine import Connection-from skyvern.config import settingsfrom skyvern.forge.sdk.workflow.sequential_key import resolve_reuse_bound_key@@ -116,84 +114,70 @@ class _ConnectionOp:def test_binding_migration_replaces_invalid_concurrent_index_on_postgres(- monkeypatch: pytest.MonkeyPatch,+ monkeypatch: pytest.MonkeyPatch, postgres_scratch_database: Engine) -> None:- database_url = make_url(str(settings.DATABASE_STRING))- if database_url.get_backend_name() != "postgresql":- pytest.skip("requires PostgreSQL")- database_url = database_url.set(drivername="postgresql+psycopg")- engine = create_engine(database_url, isolation_level="AUTOCOMMIT")- schema = f"reuse_migration_{uuid.uuid4().hex}"+ engine = postgres_scratch_database.execution_options(isolation_level="AUTOCOMMIT")migration = _load_migration_module()- try:- with engine.connect() as connection:- connection.exec_driver_sql(f'CREATE SCHEMA "{schema}"')- connection.exec_driver_sql(f'SET search_path TO "{schema}"')- connection.exec_driver_sql(- "CREATE TABLE persistent_browser_sessions ("- "id INTEGER PRIMARY KEY, "- "organization_id VARCHAR NOT NULL, "- "bound_workflow_permanent_id VARCHAR, "- "bound_key VARCHAR, "- "deleted_at TIMESTAMP, "- "status VARCHAR NOT NULL)"- )+ with engine.connect() as connection:+ connection.exec_driver_sql(+ "CREATE TABLE persistent_browser_sessions ("+ "id INTEGER PRIMARY KEY, "+ "organization_id VARCHAR NOT NULL, "+ "bound_workflow_permanent_id VARCHAR, "+ "bound_key VARCHAR, "+ "deleted_at TIMESTAMP, "+ "status VARCHAR NOT NULL)"+ )+ connection.exec_driver_sql(+ "INSERT INTO persistent_browser_sessions "+ "(id, organization_id, bound_workflow_permanent_id, bound_key, status) VALUES "+ "(1, 'org', 'wpid', NULL, 'running'), "+ "(2, 'org', 'wpid', NULL, 'running')"+ )+ with pytest.raises(Exception):connection.exec_driver_sql(- "INSERT INTO persistent_browser_sessions "- "(id, organization_id, bound_workflow_permanent_id, bound_key, status) VALUES "- "(1, 'org', 'wpid', NULL, 'running'), "- "(2, 'org', 'wpid', NULL, 'running')"- )- with pytest.raises(Exception):- connection.exec_driver_sql(- "CREATE UNIQUE INDEX CONCURRENTLY uq_pbs_live_workflow_binding "- "ON persistent_browser_sessions "- "(organization_id, bound_workflow_permanent_id, COALESCE(bound_key, '')) "- "WHERE bound_workflow_permanent_id IS NOT NULL "- "AND deleted_at IS NULL "- "AND status IN ('created', 'running', 'retry')"- )- connection.rollback()- invalid = connection.exec_driver_sql(- "SELECT indisvalid FROM pg_catalog.pg_index "- "WHERE indexrelid = to_regclass('uq_pbs_live_workflow_binding')"- ).scalar_one()- assert invalid is False- connection.exec_driver_sql("DELETE FROM persistent_browser_sessions WHERE id = 2")-- monkeypatch.setattr(migration, "op", _ConnectionOp(connection))- migration.upgrade()-- valid, unique = connection.exec_driver_sql(- "SELECT indisvalid, indisunique FROM pg_catalog.pg_index "- "WHERE indexrelid = to_regclass('uq_pbs_live_workflow_binding')"- ).one()- assert valid is True- assert unique is True- oversized_key, _ = resolve_reuse_bound_key(- SimpleNamespace(- workflow_definition=SimpleNamespace(blocks=[]),- browser_profile_key="{{ payload }}",- sequential_key=None,- workflow_permanent_id="wpid",- ),- {"payload": "x" * 4096},- {},- )- assert oversized_key.startswith("profile:sha256:")- connection.exec_driver_sql(- "INSERT INTO persistent_browser_sessions "- "(id, organization_id, bound_workflow_permanent_id, bound_key, status) "- "VALUES (3, 'org', 'wpid', %s, 'running')",- (oversized_key,),- )- assert (- connection.exec_driver_sql(- "SELECT bound_key FROM persistent_browser_sessions WHERE id = 3"- ).scalar_one()- == oversized_key+ "CREATE UNIQUE INDEX CONCURRENTLY uq_pbs_live_workflow_binding "+ "ON persistent_browser_sessions "+ "(organization_id, bound_workflow_permanent_id, COALESCE(bound_key, '')) "+ "WHERE bound_workflow_permanent_id IS NOT NULL "+ "AND deleted_at IS NULL "+ "AND status IN ('created', 'running', 'retry')")- connection.exec_driver_sql(f'DROP SCHEMA "{schema}" CASCADE')- finally:- engine.dispose()+ connection.rollback()+ invalid = connection.exec_driver_sql(+ "SELECT indisvalid FROM pg_catalog.pg_index WHERE indexrelid = to_regclass('uq_pbs_live_workflow_binding')"+ ).scalar_one()+ assert invalid is False+ connection.exec_driver_sql("DELETE FROM persistent_browser_sessions WHERE id = 2")++ monkeypatch.setattr(migration, "op", _ConnectionOp(connection))+ migration.upgrade()++ valid, unique = connection.exec_driver_sql(+ "SELECT indisvalid, indisunique FROM pg_catalog.pg_index "+ "WHERE indexrelid = to_regclass('uq_pbs_live_workflow_binding')"+ ).one()+ assert valid is True+ assert unique is True+ oversized_key, _ = resolve_reuse_bound_key(+ SimpleNamespace(+ workflow_definition=SimpleNamespace(blocks=[]),+ browser_profile_key="{{ payload }}",+ sequential_key=None,+ workflow_permanent_id="wpid",+ ),+ {"payload": "x" * 4096},+ {},+ )+ assert oversized_key.startswith("profile:sha256:")+ connection.exec_driver_sql(+ "INSERT INTO persistent_browser_sessions "+ "(id, organization_id, bound_workflow_permanent_id, bound_key, status) "+ "VALUES (3, 'org', 'wpid', %s, 'running')",+ (oversized_key,),+ )+ assert (+ connection.exec_driver_sql("SELECT bound_key FROM persistent_browser_sessions WHERE id = 3").scalar_one()+ == oversized_key+ )