Skip to content

armgit / ARES

Public Imported from GitHub
0

ARES - authorized red-team engagement automation with dashboard, campaign scope, module orchestration, OPSEC controls, encrypted vault, and reporting.

feat(db): add verified schema adoption

Mafifrizi643af7888a90
diff --git a/QUICKSTART.md b/QUICKSTART.md
index e739f56..8a9939a 100644
--- a/QUICKSTART.md
+++ b/QUICKSTART.md
@@ -438,3 +438,10 @@ reports, and API key lifecycle.
ARES is intended for authorized security testing, lab use, and defensive
validation only.
+## Database ownership
+
+Fresh databases initialize through Alembic. If ARES reports that an existing
+SQLite or PostgreSQL database requires adoption, stop all workers and follow
+[`docs/database-migrations.md`](docs/database-migrations.md). Do not run a blind
+`alembic stamp`: only audited unversioned generations 6 and 7 are supported, and
+SQLite adoption requires a retained verified backup.
diff --git a/README.md b/README.md
index 429add5..73ac5ff 100644
--- a/README.md
+++ b/README.md
@@ -620,7 +620,12 @@ cd frontend && npm run build
```
Do not commit `.env`, `.venv`, local databases, reports with real client data,
-API keys, tokens, or screenshots that expose secrets.
+API keys, tokens, screenshots that expose secrets, or database-adoption backups.
+Persistent databases are Alembic-managed. Existing
+unversioned generation-6/7 databases require the explicit verified adoption
+workflow; startup never stamps them automatically. See
+[`docs/database-migrations.md`](docs/database-migrations.md) before upgrading a
+deployment.
---
diff --git a/ares/db/database.py b/ares/db/database.py
index 6d61ea5..f7f3377 100644
--- a/ares/db/database.py
+++ b/ares/db/database.py
@@ -693,6 +693,53 @@ class AresDatabase:
return await db.connect()
async def _init_schema(self) -> None:
+ if not self._is_sqlite_uri:
+ from ares.db.migrations.adoption import (
+ AdoptionExit,
+ AdoptionFailure,
+ inspect_sqlite_database,
+ )
+
+ try:
+ ownership = await asyncio.to_thread(
+ inspect_sqlite_database,
+ self._db_path,
+ )
+ except AdoptionFailure as exc:
+ if exc.exit_code == AdoptionExit.MANAGED_METADATA:
+ raise RuntimeError(
+ "Invalid managed SQLite database metadata"
+ ) from None
+ raise RuntimeError("Incompatible SQLite database catalog") from None
+
+ if ownership.diagnostic.startswith("ARES-M2B-ADOPTION-READY:"):
+ raise RuntimeError("SQLite database adoption is required")
+ if ownership.diagnostic == "ARES-M2B-ALREADY-MANAGED:0008":
+ return
+ if ownership.exit_code == AdoptionExit.MIGRATION_REQUIRED:
+ if ownership.predecessor not in {
+ f"{revision:04d}" for revision in range(1, 8)
+ }:
+ raise RuntimeError(
+ "Invalid managed SQLite database metadata"
+ )
+ elif ownership.diagnostic != "ARES-M2B-EMPTY-DATABASE":
+ raise RuntimeError("Incompatible SQLite database catalog")
+
+ if await self._run_alembic_migrations():
+ try:
+ managed = await asyncio.to_thread(
+ inspect_sqlite_database,
+ self._db_path,
+ )
+ except AdoptionFailure:
+ raise RuntimeError(
+ "SQLite managed initialization failed"
+ ) from None
+ if managed.diagnostic != "ARES-M2B-ALREADY-MANAGED:0008":
+ raise RuntimeError("SQLite managed initialization failed")
+ return
+
async with self._conn.execute(
"SELECT 1 FROM sqlite_master "
"WHERE type='table' AND name='websocket_tickets'"
@@ -1067,25 +1114,28 @@ class AresDatabase:
return False
try:
- from pathlib import Path
- from alembic.config import Config as AlembicConfig
+ from types import SimpleNamespace
+
from alembic import command as alembic_command
+ from alembic.config import Config as AlembicConfig
- repo_root = Path(__file__).parent.parent.parent
- alembic_ini = repo_root / "alembic.ini"
- if not alembic_ini.exists():
- logger.debug("alembic_ini_not_found", file="alembic.ini")
- return False
+ from ares.db.migrations.adoption import migration_config
- alembic_cfg = AlembicConfig(str(alembic_ini))
db_url = f"sqlite:///{self._db_path}"
- alembic_cfg.set_main_option("sqlalchemy.url", db_url)
- import asyncio
loop = asyncio.get_running_loop()
+
+ def _upgrade() -> None:
+ with migration_config() as configured:
+ alembic_cfg: AlembicConfig = configured
+ alembic_cfg.cmd_opts = SimpleNamespace(
+ x=[f"db_url={db_url}"]
+ )
+ alembic_command.upgrade(alembic_cfg, "head")
+
await loop.run_in_executor(
None,
- lambda: alembic_command.upgrade(alembic_cfg, "head")
+ _upgrade,
)
logger.info(
"alembic_migrations_applied",
@@ -1094,15 +1144,12 @@ class AresDatabase:
return True
except ImportError:
- logger.debug("alembic_not_installed", hint="pip install alembic")
- return False
+ raise RuntimeError("SQLite managed initialization failed") from None
except Exception as exc:
- logger.warning(
- "alembic_migration_failed",
- error_type=type(exc).__name__,
- fallback="raw_sql_create_if_not_exists",
- )
- return False
+ raise RuntimeError(
+ "SQLite managed initialization failed "
+ f"[{type(exc).__name__}]"
+ ) from None
# ── Backup / export ───────────────────────────────────────────────────────
diff --git a/ares/db/migrations/__init__.py b/ares/db/migrations/__init__.py
index aab0427..0a5ccbc 100644
--- a/ares/db/migrations/__init__.py
+++ b/ares/db/migrations/__init__.py
@@ -1,19 +1,20 @@
-"""
-ARES Database Migrations — Alembic is canonical.
-
-This package previously contained a custom migration runner (runner.py).
-That has been removed. Alembic in migrations/ at repo root is the single
-source of truth for all schema changes.
+"""Explicit operator tooling for canonical Alembic database ownership.
-To run migrations:
- alembic upgrade head # apply all pending
- alembic downgrade -1 # roll back one step
- alembic revision -m "add_x" # create a new migration
-
-AresDatabase.connect() runs `alembic upgrade head` automatically at startup.
-See: migrations/versions/ for the migration chain.
+Application startup never invokes the adoption commands in this package.
+Use ``python -m ares.db.migrations --help`` for the fixed noninteractive
+operator surface.
"""
-# Nothing to import — Alembic is invoked programmatically via database.py
-# No public exports — Alembic is invoked programmatically.
-__all__: list = []
+from ares.db.migrations.adoption import (
+ SUPPORTED_GENERATIONS,
+ AdoptionExit,
+ AdoptionResult,
+ RuntimeGeneration,
+)
+
+__all__ = [
+ "AdoptionExit",
+ "AdoptionResult",
+ "RuntimeGeneration",
+ "SUPPORTED_GENERATIONS",
+]
diff --git a/ares/db/migrations/__main__.py b/ares/db/migrations/__main__.py
new file mode 100644
index 0000000..e86eb6d
--- /dev/null
+++ b/ares/db/migrations/__main__.py
@@ -0,0 +1,74 @@
+"""Noninteractive operator interface for verified database adoption."""
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from ares.db.migrations.adoption import AdoptionExit, safe_execute
+
+
+class _HelpRequestedError(Exception):
+ pass
+
+
+class _FixedParser(argparse.ArgumentParser):
+ def error(self, _message: str) -> None:
+ raise ValueError
+
+ def exit(self, status: int = 0, message: str | None = None) -> None:
+ if status == 0:
+ raise _HelpRequestedError
+ raise ValueError
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = _FixedParser(prog="python -m ares.db.migrations")
+ subparsers = parser.add_subparsers(dest="operation")
+ subparsers.add_parser("verify-adoption")
+ subparsers.add_parser("verify-managed")
+
+ adopt = subparsers.add_parser("adopt")
+ adopt.add_argument("--confirm-adoption", action="store_true")
+ adopt.add_argument("--confirm-external-backup", action="store_true")
+ adopt.add_argument("--sqlite-backup")
+
+ restore = subparsers.add_parser("restore-sqlite")
+ restore.add_argument("--confirm-restore", action="store_true")
+ restore.add_argument("--sqlite-backup")
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ parser = _parser()
+ try:
+ args = parser.parse_args(argv)
+ except _HelpRequestedError:
+ return int(AdoptionExit.OK)
+ except (SystemExit, ValueError):
+ print("ARES-M2B-E02:USAGE")
+ return int(AdoptionExit.USAGE)
+ if args.operation is None:
+ parser.print_help()
+ return int(AdoptionExit.OK)
+ if args.operation == "adopt" and not args.confirm_adoption:
+ print("ARES-M2B-E02:USAGE")
+ return int(AdoptionExit.USAGE)
+ if args.operation == "restore-sqlite" and not args.confirm_restore:
+ print("ARES-M2B-E02:USAGE")
+ return int(AdoptionExit.USAGE)
+ result = safe_execute(
+ args.operation,
+ sqlite_backup=getattr(args, "sqlite_backup", None),
+ external_backup_confirmed=getattr(
+ args,
+ "confirm_external_backup",
+ False,
+ ),
+ )
+ print(result.diagnostic)
+ return int(result.exit_code)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/ares/db/migrations/adoption.py b/ares/db/migrations/adoption.py
new file mode 100644
index 0000000..473f8b7
--- /dev/null
+++ b/ares/db/migrations/adoption.py
@@ -0,0 +1,879 @@
+"""Verified, explicit adoption of supported unversioned ARES databases.
+
+This module is intentionally independent of application startup. It recognizes
+only the frozen runtime generations below, creates no version metadata until a
+complete read-only verification succeeds, and drives the canonical Alembic
+history through one caller-owned connection.
+"""
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import ctypes
+import hashlib
+import json
+import os
+import sqlite3
+from collections.abc import Iterator
+from dataclasses import dataclass
+from enum import IntEnum
+from importlib import import_module, resources
+from pathlib import Path
+from typing import Any, Final
+
+import sqlalchemy as sa
+from alembic import command
+from alembic.config import Config
+from alembic.script import ScriptDirectory
+from sqlalchemy.engine import URL, Connection, make_url
+from sqlalchemy.ext.asyncio import create_async_engine
+
+
+class AdoptionExit(IntEnum):
+ OK = 0
+ USAGE = 2
+ CONFIGURATION = 3
+ MIGRATION_REQUIRED = 4
+ MANAGED_METADATA = 5
+ UNRECOGNIZED = 6
+ OWNERSHIP = 7
+ BACKUP = 8
+ ROLLED_BACK = 9
+ RECOVERY_REQUIRED = 10
+ POST_VERIFY = 11
+
+
+@dataclass(frozen=True, slots=True, repr=False, eq=False)
+class RuntimeGeneration:
+ dialect: str
+ generation: int
+ predecessor: str
+ has_websocket_tickets: bool
+
+
+@dataclass(frozen=True, slots=True, repr=False, eq=False)
+class AdoptionResult:
+ exit_code: AdoptionExit
+ diagnostic: str
+ predecessor: str | None = None
+
+
+SUPPORTED_GENERATIONS: Final[tuple[RuntimeGeneration, ...]] = (
+ RuntimeGeneration("sqlite", 6, "0006", False),
+ RuntimeGeneration("sqlite", 7, "0007", True),
+ RuntimeGeneration("postgresql", 6, "0006", False),
+ RuntimeGeneration("postgresql", 7, "0007", True),
+)
+
+_KNOWN_REVISIONS: Final[frozenset[str]] = frozenset(
+ f"{value:04d}" for value in range(1, 9)
+)
+_REVISION_FILES: Final[tuple[str, ...]] = (
+ "0001_initial_schema.py",
+ "0002_add_cvss_to_findings.py",
+ "0003_add_rate_limit_tracking.py",
+ "0004_add_revoked_access_tokens.py",
+ "0005_add_trace_id_to_findings.py",
+ "0006_rename_cracked_value_to_enc.py",
+ "0007_add_websocket_tickets.py",
+ "0008_reconcile_schema_parity.py",
+)
+_LOCK_KEY: Final[int] = int.from_bytes(
+ hashlib.sha256(b"ARES-M2B-ADOPTION-LOCK-V1").digest()[:8],
+ byteorder="big",
+ signed=True,
+)
+_FIXED_DIAGNOSTICS: Final[frozenset[str]] = frozenset(
+ {
+ "ARES-M2B-EMPTY-DATABASE",
+ "ARES-M2B-ALREADY-MANAGED:0008",
+ "ARES-M2B-MIGRATION-REQUIRED",
+ "ARES-M2B-ADOPTED:SQLITE:0008",
+ "ARES-M2B-ADOPTED:POSTGRESQL:0008",
+ "ARES-M2B-RESTORED:SQLITE",
+ "ARES-M2B-E02:USAGE",
+ "ARES-M2B-E03:CONFIGURATION",
+ "ARES-M2B-E04:MIGRATION-REQUIRED",
+ "ARES-M2B-E05:MANAGED-METADATA",
+ "ARES-M2B-E06:UNRECOGNIZED-CATALOG",
+ "ARES-M2B-E07:OWNERSHIP-BUSY",
+ "ARES-M2B-E08:BACKUP-FAILED",
+ "ARES-M2B-E09:ADOPTION-ROLLED-BACK",
+ "ARES-M2B-E10:RECOVERY-REQUIRED",
+ "ARES-M2B-E11:POST-VERIFY-FAILED",
+ }
+)
+
+
+class AdoptionFailure(RuntimeError): # noqa: N818 - public fixed-boundary contract
+ """Fixed, sanitized failure raised only inside the adoption boundary."""
+
+ def __init__(self, exit_code: AdoptionExit, diagnostic: str) -> None:
+ if diagnostic not in _FIXED_DIAGNOSTICS and not diagnostic.startswith(
+ "ARES-M2B-ADOPTION-READY:"
+ ):
+ raise ValueError("Invalid adoption diagnostic")
+ super().__init__(diagnostic)
+ self.exit_code = exit_code
+ self.diagnostic = diagnostic
+
+
+def _failure(exit_code: AdoptionExit, diagnostic: str) -> AdoptionFailure:
+ return AdoptionFailure(exit_code, diagnostic)
+
+
+def _revision_module() -> Any:
+ return import_module("migrations.versions.0008_reconcile_schema_parity")
+
+
+def migration_config(connection: Connection | None = None) -> Iterator[Config]:
+ """Materialize and validate the one installed canonical migration tree."""
+ root = resources.files("migrations")
+ with resources.as_file(root) as migration_root:
+ required = {
+ "__init__.py",
+ "env.py",
+ "script.py.mako",
+ }
+ if any(not (migration_root / name).is_file() for name in required):
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+ versions = migration_root / "versions"
+ actual_revisions = tuple(
+ sorted(path.name for path in versions.glob("*.py"))
+ )
+ if actual_revisions != _REVISION_FILES:
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+ cfg = Config()
+ cfg.set_main_option("script_location", os.fspath(migration_root))
+ if connection is not None:
+ cfg.attributes["connection"] = connection
+ script = ScriptDirectory.from_config(cfg)
+ if script.get_heads() != ["0008"]:
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+ revisions = tuple(script.walk_revisions(base="base", head="heads"))
+ if tuple(item.revision for item in reversed(revisions)) != tuple(
+ f"{value:04d}" for value in range(1, 9)
+ ):
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+ yield cfg
+
+
+def _version_revision(connection: Connection) -> str | None:
+ revision_module = _revision_module()
+ dialect = connection.dialect.name
+ try:
+ if dialect == "sqlite":
+ objects = connection.exec_driver_sql(
+ "SELECT type FROM sqlite_schema "
+ "WHERE name='alembic_version' OR tbl_name='alembic_version'"
+ ).fetchall()
+ if not objects:
+ return None
+ rows = connection.exec_driver_sql(
+ "SELECT version_num FROM main.alembic_version"
+ ).fetchall()
+ elif dialect == "postgresql":
+ objects = connection.execute(
+ sa.text(
+ """
+ SELECT relation.relkind::text
+ FROM pg_class AS relation
+ JOIN pg_namespace AS namespace
+ ON namespace.oid=relation.relnamespace
+ WHERE namespace.nspname=current_schema()
+ AND relation.relname='alembic_version'
+ """
+ )
+ ).fetchall()
+ if not objects:
+ return None
+ rows = connection.exec_driver_sql(
+ 'SELECT version_num FROM "alembic_version"'
+ ).fetchall()
+ else:
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+ if len(rows) != 1 or not isinstance(rows[0][0], str):
+ raise ValueError
+ revision = rows[0][0]
+ if revision not in _KNOWN_REVISIONS:
+ raise ValueError
+ if dialect == "sqlite":
+ revision_module._require_sqlite_alembic_version_relation(
+ connection,
+ revision,
+ )
+ else:
+ revision_module._pg_validate_alembic_version_relation(
+ connection,
+ revision,
+ )
+ return revision
+ except AdoptionFailure:
+ raise
+ except (LookupError, RuntimeError, sa.exc.SQLAlchemyError, TypeError, ValueError):
+ raise _failure(
+ AdoptionExit.MANAGED_METADATA,
+ "ARES-M2B-E05:MANAGED-METADATA",
+ ) from None
+
+
+def _is_empty(connection: Connection) -> bool:
+ if connection.dialect.name == "sqlite":
+ return connection.exec_driver_sql(
+ "SELECT 1 FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%' LIMIT 1"
+ ).first() is None
+ return connection.execute(
+ sa.text(
+ """
+ SELECT 1
+ FROM pg_class AS relation
+ JOIN pg_namespace AS namespace
+ ON namespace.oid=relation.relnamespace
+ WHERE namespace.nspname=current_schema()
+ AND relation.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
+ LIMIT 1
+ """
+ )
+ ).first() is None
+
+
+def inspect_connection(connection: Connection) -> AdoptionResult:
+ """Classify one catalog without mutation or external dynamic diagnostics."""
+ revision = _version_revision(connection)
+ if revision == "0008":
+ try:
+ _revision_module().verify_managed_catalog(connection)
+ except (RuntimeError, sa.exc.SQLAlchemyError):
+ raise _failure(
+ AdoptionExit.MANAGED_METADATA,
+ "ARES-M2B-E05:MANAGED-METADATA",
+ ) from None
+ return AdoptionResult(
+ AdoptionExit.OK,
+ "ARES-M2B-ALREADY-MANAGED:0008",
+ )
+ if revision is not None:
+ return AdoptionResult(
+ AdoptionExit.MIGRATION_REQUIRED,
+ "ARES-M2B-MIGRATION-REQUIRED",
+ revision,
+ )
+ if _is_empty(connection):
+ return AdoptionResult(AdoptionExit.OK, "ARES-M2B-EMPTY-DATABASE")
+ try:
+ predecessor = _revision_module().classify_unversioned_catalog(connection)
+ except (RuntimeError, sa.exc.SQLAlchemyError):
+ raise _failure(
+ AdoptionExit.UNRECOGNIZED,
+ "ARES-M2B-E06:UNRECOGNIZED-CATALOG",
+ ) from None
+ dialect = connection.dialect.name.upper()
+ return AdoptionResult(
+ AdoptionExit.OK,
+ f"ARES-M2B-ADOPTION-READY:{dialect}:{predecessor}",
+ predecessor,
+ )
+
+
+def verify_managed_connection(connection: Connection) -> AdoptionResult:
+ result = inspect_connection(connection)
+ if result.diagnostic == "ARES-M2B-ALREADY-MANAGED:0008":
+ return result
+ if result.exit_code == AdoptionExit.MIGRATION_REQUIRED:
+ raise _failure(
+ AdoptionExit.MIGRATION_REQUIRED,
+ "ARES-M2B-E04:MIGRATION-REQUIRED",
+ )
+ raise _failure(
+ AdoptionExit.POST_VERIFY,
+ "ARES-M2B-E11:POST-VERIFY-FAILED",
+ )
+
+
+def _adoption_predecessor(result: AdoptionResult) -> str:
+ if result.exit_code == AdoptionExit.MIGRATION_REQUIRED:
+ raise _failure(
+ AdoptionExit.MIGRATION_REQUIRED,
+ "ARES-M2B-E04:MIGRATION-REQUIRED",
+ )
+ if result.predecessor not in {"0006", "0007"}:
+ raise _failure(
+ AdoptionExit.UNRECOGNIZED,
+ "ARES-M2B-E06:UNRECOGNIZED-CATALOG",
+ )
+ return result.predecessor
+
+
+def _run_canonical_adoption(
+ connection: Connection,
+ predecessor: str,
+) -> None:
+ if predecessor not in {"0006", "0007"}:
+ raise _failure(
+ AdoptionExit.UNRECOGNIZED,
+ "ARES-M2B-E06:UNRECOGNIZED-CATALOG",
+ )
+ adoption_marker = "ares_m2b_adoption_transaction"
+ marked_sqlite = connection.dialect.name == "sqlite"
+ if marked_sqlite:
+ connection.info[adoption_marker] = True
+ try:
+ with migration_config(connection) as cfg:
+ command.stamp(cfg, predecessor)
+ command.upgrade(cfg, "0008")
+ finally:
+ if marked_sqlite:
+ connection.info.pop(adoption_marker, None)
+ verify_managed_connection(connection)
+
+
+def _database_url() -> str:
+ try:
+ from ares.core.config import DatabaseSettings
+
+ value = DatabaseSettings(_env_file=".env").database_url
+ if not isinstance(value, str) or not value or value != value.strip():
+ raise ValueError
+ return value
+ except (ImportError, OSError, TypeError, ValueError):
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ ) from None
+
+
+def _parsed_url(raw_url: str) -> URL:
+ try:
+ url = make_url(raw_url)
+ except (sa.exc.SQLAlchemyError, TypeError, ValueError):
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ ) from None
+ backend = url.get_backend_name()
+ if backend == "sqlite":
+ return url.set(drivername="sqlite+pysqlite")
+ if backend == "postgresql":
+ return url.set(drivername="postgresql+asyncpg")
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+
+
+def _sqlite_path(url: URL) -> Path:
+ if url.get_backend_name() != "sqlite" or not url.database:
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+ if url.query or url.database == ":memory:" or url.database.startswith("file:"):
+ raise _failure(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+ path = Path(url.database)
+ if not path.is_absolute():
+ path = Path.cwd() / path
+ if path.is_symlink():
+ raise _failure(
+ AdoptionExit.UNRECOGNIZED,
+ "ARES-M2B-E06:UNRECOGNIZED-CATALOG",
+ )
+ return path.resolve(strict=False)
+
+
+def _encode_value(value: Any) -> tuple[str, Any]:
+ if value is None:
+ return "null", None
+ if isinstance(value, bytes):
+ return "blob", value.hex()
+ if isinstance(value, bool):
+ return "integer", int(value)
+ if isinstance(value, int):
+ return "integer", value
+ if isinstance(value, float):
+ return "real", value.hex()
+ if isinstance(value, str):
+ return "text", value
+ raise _failure(
+ AdoptionExit.UNRECOGNIZED,
+ "ARES-M2B-E06:UNRECOGNIZED-CATALOG",
+ )
+
+
+def sqlite_catalog_and_row_digest(connection: sqlite3.Connection) -> bytes:
+ """Return a stable SHA-256 digest without exposing catalog or row values."""
+ digest = hashlib.sha256()
+ raw_schema_rows = connection.execute(
+ "SELECT type, name, tbl_name, sql FROM sqlite_schema "
+ "ORDER BY type, name, tbl_name"
+ ).fetchall()
+ schema_rows = tuple(
+ row
+ for row in raw_schema_rows
+ if not (row[0] == "index" and row[3] is None)
+ )
+ tables = tuple(
+ str(row[1])
+ for row in schema_rows
+ if row[0] == "table" and not str(row[1]).startswith("sqlite_")
+ )
+ payload: list[Any] = [
+ tuple(tuple(_encode_value(value) for value in row) for row in schema_rows),
+ tuple(
+ (int(row[0]), str(row[1]))
+ for row in connection.execute("PRAGMA database_list").fetchall()
+ ),
+ int(connection.execute("PRAGMA foreign_keys").fetchone()[0]),
+ ]
+ if any(row[0] == "table" and row[1] == "sqlite_sequence" for row in raw_schema_rows):
+ payload.append(
+ tuple(
+ sorted(
+ (str(row[0]), int(row[1]))
+ for row in connection.execute(
+ "SELECT name, seq FROM sqlite_sequence"
+ ).fetchall()
+ )
+ )
+ )
+ for table in sorted(tables):
+ quoted = '"' + table.replace('"', '""') + '"'
+ columns = connection.execute(f"PRAGMA table_xinfo({quoted})").fetchall()
+ foreign_keys = connection.execute(
+ f"PRAGMA foreign_key_list({quoted})"
+ ).fetchall()
+ indexes = connection.execute(f"PRAGMA index_list({quoted})").fetchall()
+ index_details: list[Any] = []
+ for row in indexes:
+ index_name = str(row[1])
+ index_quoted = '"' + index_name.replace('"', '""') + '"'
+ details = connection.execute(
+ f"PRAGMA index_xinfo({index_quoted})"
+ ).fetchall()
+ semantic_name = None if str(row[3]) in {"pk", "u"} else index_name
+ index_details.append(
+ (
+ semantic_name,
+ int(row[2]),
+ str(row[3]),
+ int(row[4]),
+ tuple(details),
+ )
+ )
+ rows = connection.execute(
+ f"SELECT * FROM {quoted}" # noqa: S608 - identifier safely quoted
+ ).fetchall()
+ encoded_rows = sorted(
+ json.dumps(
+ tuple(_encode_value(value) for value in row),
+ ensure_ascii=True,
+ separators=(",", ":"),
+ )
+ for row in rows
+ )
+ payload.append(
+ (
+ table,
+ tuple(columns),
+ tuple(foreign_keys),
+ tuple(index_details),
+ tuple(encoded_rows),
+ )
+ )
+ digest.update(
+ json.dumps(
+ payload,
+ ensure_ascii=True,
+ separators=(",", ":"),
+ default=str,
+ ).encode("utf-8")
+ )
+ return digest.digest()
+
+
+def _integrity_ok(path: Path) -> bool:
+ try:
+ connection = sqlite3.connect(path)
+ try:
+ return connection.execute("PRAGMA integrity_check").fetchall() == [
+ ("ok",)
+ ]
+ finally:
+ connection.close()
+ except sqlite3.Error:
+ return False
+
+
+def _fsync_file(path: Path) -> None:
+ with path.open("r+b") as handle:
+ os.fsync(handle.fileno())
+
+
+def _fsync_directory(path: Path) -> None:
+ if os.name != "nt":
+ descriptor = os.open(path, os.O_RDONLY)
+ try:
+ os.fsync(descriptor)
+ finally:
+ os.close(descriptor)
+ return
+ create_file = ctypes.windll.kernel32.CreateFileW
+ create_file.restype = ctypes.c_void_p
+ handle = create_file(
+ str(path),
+ 0x40000000,
+ 0x00000007,
+ None,
+ 3,
+ 0x02000000,
+ None,
+ )
+ invalid = ctypes.c_void_p(-1).value
+ if handle in {None, invalid}:
+ raise OSError
+ try:
+ if not ctypes.windll.kernel32.FlushFileBuffers(handle):
+ raise OSError
+ finally:
+ ctypes.windll.kernel32.CloseHandle(handle)
+
+
+def _durable(path: Path) -> None:
+ _fsync_file(path)
+ _fsync_directory(path.parent)
+
+
+def _create_sqlite_backup(source: Path, backup: Path) -> bytes:
+ if backup.exists() or source == backup.resolve(strict=False):
+ raise _failure(AdoptionExit.BACKUP, "ARES-M2B-E08:BACKUP-FAILED")
+ if not backup.parent.is_dir():
+ raise _failure(AdoptionExit.BACKUP, "ARES-M2B-E08:BACKUP-FAILED")
+ descriptor: int | None = None
+ try:
+ descriptor = os.open(backup, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+ os.close(descriptor)
+ descriptor = None
+ source_connection = sqlite3.connect(source)
+ backup_connection = sqlite3.connect(backup)
+ try:
+ source_connection.backup(backup_connection)
+ finally:
+ backup_connection.close()
+ source_connection.close()
+ if not _integrity_ok(backup):
+ raise OSError
+ _durable(backup)
+ verified = sqlite3.connect(backup)
+ try:
+ verified.execute("PRAGMA foreign_keys=ON")
+ return sqlite_catalog_and_row_digest(verified)
+ finally:
+ verified.close()
+ except AdoptionFailure:
+ raise
+ except (OSError, sqlite3.Error):
+ if descriptor is not None:
+ os.close(descriptor)
+ raise _failure(
+ AdoptionExit.BACKUP,
+ "ARES-M2B-E08:BACKUP-FAILED",
+ ) from None
+
+
+def _sqlite_engine(path: Path) -> sa.Engine:
+ engine = sa.create_engine(
+ URL.create("sqlite+pysqlite", database=os.fspath(path)),
+ poolclass=sa.pool.NullPool,
+ )
+
+ @sa.event.listens_for(engine, "connect")
+ def _configure(dbapi_connection: sqlite3.Connection, _record: Any) -> None:
+ dbapi_connection.execute("PRAGMA foreign_keys=ON")
+
+ return engine
+
+
+def inspect_sqlite_database(path: str | Path) -> AdoptionResult:
+ engine = _sqlite_engine(Path(path).resolve(strict=False))
+ try:
+ with engine.connect() as connection:
+ return inspect_connection(connection)
+ finally:
+ engine.dispose()
+
+
+def adopt_sqlite(path: Path, backup: Path) -> AdoptionResult:
+ initial = inspect_sqlite_database(path)
+ if initial.diagnostic == "ARES-M2B-ALREADY-MANAGED:0008":
+ return initial
+ _adoption_predecessor(initial)
+ backup_digest = _create_sqlite_backup(path, backup)
+ engine = _sqlite_engine(path)
+ rollback_attempted = False
+ rollback_proven = False
+ try:
+ with engine.connect() as connection:
+ connection.exec_driver_sql("BEGIN EXCLUSIVE")
+ raw = connection.connection.driver_connection
+ if not isinstance(raw, sqlite3.Connection):
+ raise _failure(
+ AdoptionExit.RECOVERY_REQUIRED,
+ "ARES-M2B-E10:RECOVERY-REQUIRED",
+ )
+ if sqlite_catalog_and_row_digest(raw) != backup_digest:
+ connection.rollback()
+ raise _failure(
+ AdoptionExit.BACKUP,
+ "ARES-M2B-E08:BACKUP-FAILED",
+ )
+ classified = inspect_connection(connection)
+ predecessor = _adoption_predecessor(classified)
+ try:
+ _run_canonical_adoption(connection, predecessor)
+ connection.commit()
+ except BaseException:
+ connection.rollback()
+ rollback_attempted = True
+ try:
+ raw.execute("PRAGMA foreign_keys=ON")
+ foreign_keys = int(
+ raw.execute("PRAGMA foreign_keys").fetchone()[0]
+ )
+ rollback_proven = (
+ foreign_keys == 1
+ and sqlite_catalog_and_row_digest(raw) == backup_digest
+ )
+ except (LookupError, sqlite3.Error, TypeError, ValueError):
+ rollback_proven = False
+ raise
+ _durable(path)
+ return AdoptionResult(AdoptionExit.OK, "ARES-M2B-ADOPTED:SQLITE:0008")
+ except (asyncio.CancelledError, KeyboardInterrupt, SystemExit, GeneratorExit):
+ raise
+ except AdoptionFailure:
+ raise
+ except Exception:
+ if rollback_attempted and not rollback_proven:
+ try:
+ verification = sqlite3.connect(path)
+ try:
+ verification.execute("PRAGMA foreign_keys=ON")
+ rollback_proven = (
+ int(
+ verification.execute(
+ "PRAGMA foreign_keys"
+ ).fetchone()[0]
+ )
+ == 1
+ and sqlite_catalog_and_row_digest(verification)
+ == backup_digest
+ )
+ finally:
+ verification.close()
+ except (LookupError, OSError, sqlite3.Error, TypeError, ValueError):
+ rollback_proven = False
+ if rollback_proven:
+ raise _failure(
+ AdoptionExit.ROLLED_BACK,
+ "ARES-M2B-E09:ADOPTION-ROLLED-BACK",
+ ) from None
+ raise _failure(
+ AdoptionExit.RECOVERY_REQUIRED,
+ "ARES-M2B-E10:RECOVERY-REQUIRED",
+ ) from None
+ finally:
+ engine.dispose()
+
+
+def restore_sqlite(path: Path, backup: Path) -> AdoptionResult:
+ if path == backup.resolve(strict=False) or not _integrity_ok(backup):
+ raise _failure(AdoptionExit.BACKUP, "ARES-M2B-E08:BACKUP-FAILED")
+ source = sqlite3.connect(path, isolation_level=None)
+ verified_backup = sqlite3.connect(backup)
+ try:
+ source.execute("PRAGMA foreign_keys=ON")
+ locking_mode = source.execute(
+ "PRAGMA locking_mode=EXCLUSIVE"
+ ).fetchone()
+ if locking_mode != ("exclusive",):
+ raise sqlite3.OperationalError
+ source.execute("BEGIN EXCLUSIVE")
+ source.rollback()
+ verified_backup.backup(source)
+ source.commit()
+ except (OSError, sqlite3.Error):
+ raise _failure(
+ AdoptionExit.RECOVERY_REQUIRED,
+ "ARES-M2B-E10:RECOVERY-REQUIRED",
+ ) from None
+ finally:
+ verified_backup.close()
+ source.close()
+ if not _integrity_ok(path):
+ raise _failure(
+ AdoptionExit.RECOVERY_REQUIRED,
+ "ARES-M2B-E10:RECOVERY-REQUIRED",
+ )
+ _durable(path)
+ result = inspect_sqlite_database(path)
+ if result.exit_code not in {AdoptionExit.OK, AdoptionExit.MIGRATION_REQUIRED}:
+ raise _failure(
+ AdoptionExit.POST_VERIFY,
+ "ARES-M2B-E11:POST-VERIFY-FAILED",
+ )
+ return AdoptionResult(AdoptionExit.OK, "ARES-M2B-RESTORED:SQLITE")
+
+
+def _sync_inspect(sync_connection: Connection) -> AdoptionResult:
+ return inspect_connection(sync_connection)
+
+
+def _sync_adopt(sync_connection: Connection, predecessor: str) -> None:
+ _run_canonical_adoption(sync_connection, predecessor)
+
+
+async def _postgres_operation(url: URL, operation: str) -> AdoptionResult:
+ engine = create_async_engine(url, poolclass=sa.pool.NullPool)
+ already_managed = False
+ try:
+ async with engine.connect() as connection:
+ if operation == "verify-adoption":
+ return await connection.run_sync(_sync_inspect)
+ if operation == "verify-managed":
+ return await connection.run_sync(verify_managed_connection)
+ transaction = await connection.begin()
+ try:
+ locked = bool(
+ await connection.scalar(
+ sa.text("SELECT pg_try_advisory_xact_lock(:key)"),
+ {"key": _LOCK_KEY},
+ )
+ )
+ if not locked:
+ raise _failure(
+ AdoptionExit.OWNERSHIP,
+ "ARES-M2B-E07:OWNERSHIP-BUSY",
+ )
+ classified = await connection.run_sync(_sync_inspect)
+ if classified.diagnostic == "ARES-M2B-ALREADY-MANAGED:0008":
+ already_managed = True
+ else:
+ predecessor = _adoption_predecessor(classified)
+ await connection.run_sync(
+ _sync_adopt,
+ predecessor,
+ )
+ await transaction.commit()
+ except BaseException:
+ if transaction.is_active:
+ await transaction.rollback()
+ raise
+ async with engine.connect() as verification:
+ await verification.run_sync(verify_managed_connection)
+ if already_managed:
+ return AdoptionResult(
+ AdoptionExit.OK,
+ "ARES-M2B-ALREADY-MANAGED:0008",
+ )
+ return AdoptionResult(AdoptionExit.OK, "ARES-M2B-ADOPTED:POSTGRESQL:0008")
+ finally:
+ await engine.dispose()
+
+
+async def execute(
+ operation: str,
+ *,
+ sqlite_backup: str | None = None,
+ external_backup_confirmed: bool = False,
+) -> AdoptionResult:
+ """Execute one fixed operator action using canonical settings resolution."""
+ url = _parsed_url(_database_url())
+ if url.get_backend_name() == "postgresql":
+ if operation == "adopt" and not external_backup_confirmed:
+ raise _failure(AdoptionExit.USAGE, "ARES-M2B-E02:USAGE")
+ return await _postgres_operation(url, operation)
+ path = _sqlite_path(url)
+ if operation == "verify-adoption":
+ return await asyncio.to_thread(inspect_sqlite_database, path)
+ if operation == "verify-managed":
+ return await asyncio.to_thread(
+ lambda: verify_managed_connection_from_path(path)
+ )
+ if sqlite_backup is None:
+ raise _failure(AdoptionExit.USAGE, "ARES-M2B-E02:USAGE")
+ backup = Path(sqlite_backup)
+ if not backup.is_absolute():
+ backup = Path.cwd() / backup
+ backup = backup.resolve(strict=False)
+ if operation == "adopt":
+ return await asyncio.to_thread(adopt_sqlite, path, backup)
+ if operation == "restore-sqlite":
+ return await asyncio.to_thread(restore_sqlite, path, backup)
+ raise _failure(AdoptionExit.USAGE, "ARES-M2B-E02:USAGE")
+
+
+def verify_managed_connection_from_path(path: Path) -> AdoptionResult:
+ engine = _sqlite_engine(path)
+ try:
+ with engine.connect() as connection:
+ return verify_managed_connection(connection)
+ finally:
+ engine.dispose()
+
+
+def safe_execute(
+ operation: str,
+ *,
+ sqlite_backup: str | None = None,
+ external_backup_confirmed: bool = False,
+) -> AdoptionResult:
+ try:
+ return asyncio.run(
+ execute(
+ operation,
+ sqlite_backup=sqlite_backup,
+ external_backup_confirmed=external_backup_confirmed,
+ )
+ )
+ except (asyncio.CancelledError, KeyboardInterrupt, SystemExit, GeneratorExit):
+ raise
+ except AdoptionFailure as exc:
+ return AdoptionResult(exc.exit_code, exc.diagnostic)
+ except (OSError, sqlite3.Error, sa.exc.SQLAlchemyError):
+ return AdoptionResult(
+ AdoptionExit.CONFIGURATION,
+ "ARES-M2B-E03:CONFIGURATION",
+ )
+
+
+__all__ = [
+ "AdoptionExit",
+ "AdoptionResult",
+ "RuntimeGeneration",
+ "SUPPORTED_GENERATIONS",
+ "execute",
+ "inspect_connection",
+ "inspect_sqlite_database",
+ "migration_config",
+ "safe_execute",
+ "sqlite_catalog_and_row_digest",
+ "verify_managed_connection",
+]
diff --git a/docker/README.md b/docker/README.md
index 707bab4..93214b0 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -35,3 +35,12 @@ The Vite server proxies API and WebSocket traffic to `ares-api`.
## Environment
Copy `.env.example` to `.env` and fill in secrets before starting either stack.
+## Existing database adoption
+
+Containers do not adopt unversioned databases during startup. Before replacing
+an existing deployment, stop every application worker, take the required backup,
+and run the explicit verification/adoption workflow described in
+[`docs/database-migrations.md`](../docs/database-migrations.md). PostgreSQL
+adoption is serialized with a transaction-scoped advisory lock; SQLite adoption
+requires exclusive ownership and a new durable backup target. Keep additive
+migrations installed if the application image is rolled back.
diff --git a/docs/database-migrations.md b/docs/database-migrations.md
new file mode 100644
index 0000000..e6a3a3f
--- /dev/null
+++ b/docs/database-migrations.md
@@ -0,0 +1,110 @@
+# Database migrations and verified adoption
+
+Alembic is the canonical schema history for every persistent ARES database.
+Fresh databases are initialized as managed databases and current managed
+databases are validated at startup. Application startup never stamps or adopts
+an existing unversioned catalog.
+
+## Managed and unversioned databases
+
+A managed database has one valid `alembic_version` relation and exactly one
+known revision row. Revision `0008` is current and forward-only. Managed
+revisions `0001` through `0007` use normal Alembic migration; malformed or
+unknown version metadata is rejected.
+
+An unversioned database has no Alembic version relation. The adoption verifier
+accepts only the complete, audited runtime-bootstrap generations 6 and 7. It
+does not use `schema_version`, table presence, or partial column matching as
+authority. Unknown or locally modified catalogs require owner-reviewed manual
+recovery and are not stamped.
+
+Stop all ARES workers before verification, adoption, or SQLite restoration.
+Use the same configured `ARES_DATABASE_URL` or `.env` used by the deployment;
+never put a database URL or credential on the command line.
+
+## Verification-only workflow
+
+Run the read-only classifier first:
+
+```text
+python -m ares.db.migrations verify-adoption
+```
+
+A supported unversioned catalog reports a fixed readiness identifier ending in
+its proven predecessor (`0006` or `0007`). A current managed database reports
+`ARES-M2B-ALREADY-MANAGED:0008`. No version relation is created by verification.
+
+After adoption or normal migration, verify the managed contract:
+
+```text
+python -m ares.db.migrations verify-managed
+```
+
+## SQLite adoption and recovery
+
+SQLite adoption requires exclusive ownership and a new backup destination:
+
+```text
+python -m ares.db.migrations adopt --confirm-adoption --sqlite-backup <backup-file>
+```
+
+The backup is created with SQLite's online backup API, independently checked,
+and durably synchronized before the source is locked or changed. The source and
+backup are compared under an exclusive lock. Adoption stamps only the proven
+predecessor and then runs normal Alembic upgrades on the same connection. The
+backup is retained after success and after failure; it is never overwritten or
+silently deleted.
+
+If the command reports `ARES-M2B-E10:RECOVERY-REQUIRED`, do not start ARES.
+Preserve both files and restore only after establishing exclusive ownership:
+
+```text
+python -m ares.db.migrations restore-sqlite --confirm-restore --sqlite-backup <backup-file>
+```
+
+Restoration does not delete the backup. A failed application rollout after a
+successful adoption normally leaves the additive managed schema installed. Use
+the verified backup only when the database itself must be returned to the
+pre-adoption state.
+
+## PostgreSQL adoption
+
+Take and verify an external database backup, stop all application workers, and
+run:
+
+```text
+python -m ares.db.migrations adopt --confirm-adoption --confirm-external-backup
+```
+
+PostgreSQL adoption uses one connection, one explicit transaction, and a stable
+transaction-scoped advisory lock. Complete verification occurs under that lock;
+the proven predecessor is stamped and upgraded to `0008` on the same
+connection. DDL, data, and version metadata roll back together on failure.
+Concurrent adoption reports a fixed ownership-busy result. Objects in unrelated
+schemas are not changed; unexpected objects in the selected schema are rejected.
+
+For multi-worker deployments, keep every worker stopped through backup,
+verification, adoption, post-adoption verification, and the application
+restart. A committed adoption is idempotent: rerunning verification reports the
+already-managed result.
+
+## Exit codes
+
+| Code | Meaning |
+| ---: | --- |
+| 0 | Verified, adopted, restored, or already current |
+| 2 | Invalid command or missing confirmation |
+| 3 | Fixed configuration or connection failure |
+| 4 | Managed revision requires normal migration |
+| 5 | Malformed or unknown managed metadata |
+| 6 | Unknown, ambiguous, or unsafe unversioned catalog |
+| 7 | Exclusive ownership or advisory lock unavailable |
+| 8 | Backup creation, durability, or verification failed |
+| 9 | Adoption failed and pristine rollback was proven |
+| 10 | Recovery state is indeterminate; startup is prohibited |
+| 11 | Post-adoption managed verification failed |
+
+Diagnostics are fixed identifiers and never contain database URLs, credentials,
+paths, schema identities, SQL, rows, digests, or exception text. Direct or blind
+`alembic stamp 0008` is unsupported. Application rollback does not remove
+additive schema or migration history.
diff --git a/migrations/env.py b/migrations/env.py
index f3fb5f5..ebefe8a 100644
--- a/migrations/env.py
+++ b/migrations/env.py
@@ -11,7 +11,7 @@ from typing import Any
from alembic import context
from sqlalchemy import event, pool
-from sqlalchemy.engine import URL, make_url
+from sqlalchemy.engine import URL, Connection, make_url
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import async_engine_from_config
@@ -178,7 +178,15 @@ async def run_async_migrations() -> None:
def run_migrations_online() -> None:
"""Run the online migration and sanitize operational diagnostics."""
try:
- asyncio.run(run_async_migrations())
+ supplied_connection = config.attributes.get("connection")
+ if supplied_connection is not None:
+ if not isinstance(supplied_connection, Connection):
+ raise _AlembicSanitizedError(
+ "Invalid caller-owned Alembic connection"
+ )
+ do_run_migrations(supplied_connection)
+ else:
+ asyncio.run(run_async_migrations())
except (
asyncio.CancelledError,
KeyboardInterrupt,
diff --git a/migrations/versions/0008_reconcile_schema_parity.py b/migrations/versions/0008_reconcile_schema_parity.py
index 83510b8..0d78ba7 100644
--- a/migrations/versions/0008_reconcile_schema_parity.py
+++ b/migrations/versions/0008_reconcile_schema_parity.py
@@ -573,7 +573,10 @@ def _require_no_sqlite_temp_shadows(bind: Any) -> None:
raise RuntimeError(_CATALOG_ERROR)
-def _require_sqlite_alembic_version_relation(bind: Any) -> None:
+def _require_sqlite_alembic_version_relation(
+ bind: Any,
+ expected_revision: str = "0007",
+) -> None:
rows = bind.execute(
sa.text(
"SELECT type, name, tbl_name, sql FROM sqlite_schema "
@@ -633,7 +636,7 @@ def _require_sqlite_alembic_version_relation(bind: Any) -> None:
"SELECT version_num FROM main.alembic_version"
).fetchall()
)
- if values != ("0007",):
+ if values != (expected_revision,):
raise RuntimeError(_CATALOG_ERROR)
@@ -1264,10 +1267,17 @@ def _require_ticket_contract(bind: Any) -> None:
raise RuntimeError(_CATALOG_ERROR)
-def _sqlite_preflight(bind: Any) -> bool:
+def _sqlite_preflight(
+ bind: Any,
+ *,
+ expected_revision: str = "0007",
+ require_version: bool = True,
+ include_ticket: bool = True,
+) -> bool:
_require_no_sqlite_temp_shadows(bind)
_require_sqlite_managed_name_ownership(bind)
- _require_sqlite_alembic_version_relation(bind)
+ if require_version:
+ _require_sqlite_alembic_version_relation(bind, expected_revision)
tables = set(sa.inspect(bind).get_table_names())
required = {
"campaigns",
@@ -1280,8 +1290,9 @@ def _sqlite_preflight(bind: Any) -> bool:
"api_keys",
"refresh_tokens",
"revoked_access_tokens",
- "websocket_tickets",
}
+ if include_ticket:
+ required.add("websocket_tickets")
if not required.issubset(tables):
raise RuntimeError(_CATALOG_ERROR)
if any(table.startswith(_TEMP_PREFIX) for table in tables):
@@ -1309,7 +1320,8 @@ def _sqlite_preflight(bind: Any) -> bool:
_require_sqlite_primary_key_index(bind, table)
_require_not_null_data(bind, table)
- _require_ticket_contract(bind)
+ if include_ticket:
+ _require_ticket_contract(bind)
_require_no_orphans(bind)
_require_boolean_integer_data(bind, tables)
_require_valid_sqlite_timestamps(bind, tables)
@@ -2064,7 +2076,8 @@ def _sqlite_upgrade(bind: Any) -> None:
# transaction, but SQLite does not emit BEGIN for DDL. An explicit
# write transaction makes the reconciliation and Alembic's subsequent
# version-row update one atomic unit. Alembic owns commit/rollback.
- bind.exec_driver_sql("BEGIN IMMEDIATE")
+ if not bind.info.get("ares_m2b_adoption_transaction", False):
+ bind.exec_driver_sql("BEGIN IMMEDIATE")
exact = _sqlite_preflight(bind)
if exact:
return
@@ -2413,7 +2426,10 @@ def _pg_check_name(table: str, column: str) -> str:
return f"ck_{table}_{column}_finite"
-def _pg_validate_alembic_version_relation(bind: Any) -> None:
+def _pg_validate_alembic_version_relation(
+ bind: Any,
+ expected_revision: str = "0007",
+) -> None:
relation_rows = list(
bind.execute(
sa.text(
@@ -2821,7 +2837,7 @@ def _pg_validate_alembic_version_relation(bind: Any) -> None:
f'SELECT version_num FROM {quoted_schema}."alembic_version"' # noqa: S608
).fetchall()
)
- if values != ("0007",):
+ if values != (expected_revision,):
raise RuntimeError(_CATALOG_ERROR)
@@ -2889,7 +2905,10 @@ def _pg_relation_rows(bind: Any) -> list[dict[str, object]]:
)
-def _pg_validate_relation_shapes(bind: Any) -> set[str]:
+def _pg_validate_relation_shapes(
+ bind: Any,
+ required_tables: frozenset[str] | None = None,
+) -> set[str]:
rows = _pg_relation_rows(bind)
relations = {
str(row["table_name"]): row
@@ -2898,9 +2917,10 @@ def _pg_validate_relation_shapes(bind: Any) -> set[str]:
if len(relations) != len(rows):
raise RuntimeError(_CATALOG_ERROR)
tables = set(relations)
- if not _PG_REQUIRED_TABLES.issubset(tables):
+ required = required_tables or _PG_REQUIRED_TABLES
+ if not required.issubset(tables):
raise RuntimeError(_CATALOG_ERROR)
- if not tables.issubset(_PG_REQUIRED_TABLES | _PG_OPTIONAL_TABLES):
+ if not tables.issubset(required | _PG_OPTIONAL_TABLES):
raise RuntimeError(_CATALOG_ERROR)
expected = ("r", "p", False, False, False, 0, 0, 0, 0, 0)
for row in relations.values():
@@ -3711,6 +3731,8 @@ def _expected_pg_primary(columns: tuple[str, ...]) -> str:
def _pg_constraint_plan(
bind: Any,
tables: set[str],
+ *,
+ include_ticket: bool = True,
) -> tuple[
list[tuple[str, str, str]],
list[tuple[str, str, str, str, str]],
@@ -3869,10 +3891,11 @@ def _pg_constraint_plan(
] = _normalize_pg_definition(
"CHECK (blocked = ANY (ARRAY[0, 1]))"
)
- for name, definition in _PG_TICKET_CHECKS.items():
- expected_checks[("websocket_tickets", name)] = (
- _normalize_pg_definition(definition)
- )
+ if include_ticket:
+ for name, definition in _PG_TICKET_CHECKS.items():
+ expected_checks[("websocket_tickets", name)] = (
+ _normalize_pg_definition(definition)
+ )
for (table, name), expected in expected_checks.items():
row = next(
@@ -3941,9 +3964,21 @@ def _pg_constraint_plan(
return drop, add_fks, add_checks, add_uniques, rename_uniques
-def _postgresql_upgrade(bind: Any) -> None:
- _pg_validate_alembic_version_relation(bind)
- tables = _pg_validate_relation_shapes(bind)
+def _postgresql_reconciliation_plan(
+ bind: Any,
+ *,
+ expected_revision: str = "0007",
+ require_version: bool = True,
+ include_ticket: bool = True,
+) -> dict[str, Any]:
+ if require_version:
+ _pg_validate_alembic_version_relation(bind, expected_revision)
+ required_tables = _PG_REQUIRED_TABLES
+ if not include_ticket:
+ required_tables = frozenset(
+ table for table in required_tables if table != "websocket_tickets"
+ )
+ tables = _pg_validate_relation_shapes(bind, required_tables)
create_module_runs = "module_runs" not in tables
create_rate_limit = "rate_limit_events" not in tables
@@ -3962,7 +3997,11 @@ def _postgresql_upgrade(bind: Any) -> None:
add_checks,
add_uniques,
rename_uniques,
- ) = _pg_constraint_plan(bind, tables)
+ ) = _pg_constraint_plan(
+ bind,
+ tables,
+ include_ticket=include_ticket,
+ )
canonical_indexes = {
table: indexes
@@ -4027,6 +4066,36 @@ def _postgresql_upgrade(bind: Any) -> None:
raise RuntimeError(_CATALOG_ERROR)
drop_indexes.append(("idx_findings_validated", "findings"))
+ return {
+ "tables": tables,
+ "findings_alterations": findings_alterations,
+ "rename_credentials": rename_credentials,
+ "create_module_runs": create_module_runs,
+ "create_rate_limit": create_rate_limit,
+ "drop_constraints": drop_constraints,
+ "add_fks": add_fks,
+ "add_checks": add_checks,
+ "add_uniques": add_uniques,
+ "rename_uniques": rename_uniques,
+ "create_indexes": create_indexes,
+ "drop_indexes": drop_indexes,
+ }
+
+
+def _postgresql_upgrade(bind: Any) -> None:
+ plan = _postgresql_reconciliation_plan(bind)
+ findings_alterations = plan["findings_alterations"]
+ rename_credentials = plan["rename_credentials"]
+ create_module_runs = plan["create_module_runs"]
+ create_rate_limit = plan["create_rate_limit"]
+ drop_constraints = plan["drop_constraints"]
+ add_fks = plan["add_fks"]
+ add_checks = plan["add_checks"]
+ add_uniques = plan["add_uniques"]
+ rename_uniques = plan["rename_uniques"]
+ create_indexes = plan["create_indexes"]
+ drop_indexes = plan["drop_indexes"]
+
if rename_credentials:
op.alter_column(
"credentials",
@@ -4145,6 +4214,136 @@ def _postgresql_upgrade(bind: Any) -> None:
op.drop_index(name, table_name=table)
+def _require_postgresql_relation_set(
+ bind: Any,
+ expected_tables: set[str],
+) -> None:
+ rows = bind.execute(
+ sa.text(
+ """
+ SELECT relation.relname, relation.relkind::text AS relkind
+ FROM pg_class AS relation
+ JOIN pg_namespace AS namespace
+ ON namespace.oid=relation.relnamespace
+ WHERE namespace.nspname=current_schema()
+ AND relation.relkind IN ('r', 'p', 'v', 'm', 'f', 'S')
+ """
+ )
+ ).fetchall()
+ expected = {(table, "r") for table in expected_tables}
+ expected.add(("audit_log_id_seq", "S"))
+ if "rate_limit_events" in expected_tables:
+ expected.add(("rate_limit_events_id_seq", "S"))
+ actual = {(str(row[0]), str(row[1])) for row in rows}
+ if actual != expected:
+ raise RuntimeError(_CATALOG_ERROR)
+
+
+def classify_unversioned_catalog(bind: Any) -> str:
+ """Return the sole proven predecessor for a complete unversioned catalog."""
+ dialect = bind.dialect.name
+ if dialect == "sqlite":
+ version_objects = bind.exec_driver_sql(
+ "SELECT 1 FROM sqlite_schema "
+ "WHERE name='alembic_version' OR tbl_name='alembic_version' LIMIT 1"
+ ).first()
+ if version_objects:
+ raise RuntimeError(_CATALOG_ERROR)
+ databases = bind.exec_driver_sql("PRAGMA database_list").fetchall()
+ if len(databases) != 1 or str(databases[0][1]) != "main":
+ raise RuntimeError(_CATALOG_ERROR)
+ if int(bind.exec_driver_sql("PRAGMA foreign_keys").scalar_one()) != 1:
+ raise RuntimeError(_CATALOG_ERROR)
+ tables = {
+ str(row[0])
+ for row in bind.exec_driver_sql(
+ "SELECT name FROM sqlite_schema WHERE type='table' "
+ "AND name NOT LIKE 'sqlite_%'"
+ ).fetchall()
+ }
+ generation = "0007" if "websocket_tickets" in tables else "0006"
+ expected = set(_SQLITE_COLUMNS) - {"rate_limit_events"}
+ if generation == "0006":
+ expected.remove("websocket_tickets")
+ if tables != expected:
+ raise RuntimeError(_CATALOG_ERROR)
+ _sqlite_preflight(
+ bind,
+ require_version=False,
+ include_ticket=generation == "0007",
+ )
+ return generation
+ if dialect == "postgresql":
+ version_object = bind.execute(
+ sa.text(
+ """
+ SELECT 1
+ FROM pg_class AS relation
+ JOIN pg_namespace AS namespace
+ ON namespace.oid=relation.relnamespace
+ WHERE namespace.nspname=current_schema()
+ AND relation.relname='alembic_version'
+ LIMIT 1
+ """
+ )
+ ).first()
+ if version_object:
+ raise RuntimeError(_CATALOG_ERROR)
+ relation_names = {
+ str(row["table_name"])
+ for row in _pg_relation_rows(bind)
+ }
+ generation = (
+ "0007" if "websocket_tickets" in relation_names else "0006"
+ )
+ expected = set(_PG_REQUIRED_TABLES | _PG_OPTIONAL_TABLES) - {
+ "rate_limit_events"
+ }
+ if generation == "0006":
+ expected.remove("websocket_tickets")
+ if relation_names != expected:
+ raise RuntimeError(_CATALOG_ERROR)
+ _require_postgresql_relation_set(bind, expected)
+ _postgresql_reconciliation_plan(
+ bind,
+ require_version=False,
+ include_ticket=generation == "0007",
+ )
+ return generation
+ raise RuntimeError(_DIALECT_ERROR)
+
+
+def verify_managed_catalog(bind: Any) -> None:
+ """Verify that one selected catalog is exactly managed revision 0008."""
+ dialect = bind.dialect.name
+ if dialect == "sqlite":
+ tables = set(sa.inspect(bind).get_table_names())
+ if tables != set(_SQLITE_COLUMNS) | {"alembic_version"}:
+ raise RuntimeError(_CATALOG_ERROR)
+ if not _sqlite_preflight(bind, expected_revision="0008"):
+ raise RuntimeError(_CATALOG_ERROR)
+ return
+ if dialect == "postgresql":
+ _pg_validate_alembic_version_relation(bind, "0008")
+ expected = set(_PG_REQUIRED_TABLES | _PG_OPTIONAL_TABLES)
+ _require_postgresql_relation_set(
+ bind,
+ expected | {"alembic_version"},
+ )
+ plan = _postgresql_reconciliation_plan(
+ bind,
+ expected_revision="0008",
+ )
+ if set(plan["tables"]) != expected or any(
+ bool(value)
+ for key, value in plan.items()
+ if key != "tables"
+ ):
+ raise RuntimeError(_CATALOG_ERROR)
+ return
+ raise RuntimeError(_DIALECT_ERROR)
+
+
def upgrade() -> None:
bind = op.get_bind()
dialect = bind.dialect.name
diff --git a/pyproject.toml b/pyproject.toml
index d9345a2..e0ffb2c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -134,7 +134,10 @@ ares-setup = "ares.cli.typer_main:setup_entrypoint"
[tool.setuptools.packages.find]
where = ["."]
-include = ["ares*"]
+include = ["ares*", "migrations"]
+
+[tool.setuptools.package-data]
+migrations = ["script.py.mako", "versions/*.py"]
[tool.ruff]
line-length = 100
diff --git a/tests/integration/test_postgres_unversioned_adoption.py b/tests/integration/test_postgres_unversioned_adoption.py
new file mode 100644
index 0000000..d69dc5a
--- /dev/null
+++ b/tests/integration/test_postgres_unversioned_adoption.py
@@ -0,0 +1,360 @@
+"""Real PostgreSQL tests for explicit unversioned database adoption."""
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from uuid import uuid4
+
+import pytest
+import sqlalchemy as sa
+from sqlalchemy.engine import URL
+from sqlalchemy.ext.asyncio import create_async_engine
+
+from ares.db.migrations import adoption
+from ares.db.migrations.adoption import AdoptionExit, AdoptionFailure
+from ares.db.postgres import PostgresDatabase
+
+_ENV = (
+ "ARES_TEST_POSTGRES_HOST",
+ "ARES_TEST_POSTGRES_PORT",
+ "ARES_TEST_POSTGRES_USER",
+ "ARES_TEST_POSTGRES_DB",
+)
+_SAFE_NAME = re.compile(r"\Aares_m2b_[0-9a-f]{32}\Z")
+
+
+@dataclass(frozen=True, repr=False, eq=False)
+class _Config:
+ host: str = field(repr=False)
+ port: int = field(repr=False)
+ user: str = field(repr=False)
+ database: str = field(repr=False)
+
+
+def _config() -> _Config:
+ present = [name for name in _ENV if name in os.environ]
+ if not present:
+ pytest.skip("real PostgreSQL test environment is not configured")
+ if len(present) != len(_ENV):
+ pytest.fail("Incomplete PostgreSQL test environment", pytrace=False)
+ values = {name: os.environ[name] for name in _ENV}
+ if any(not value or value != value.strip() for value in values.values()):
+ pytest.fail("Invalid PostgreSQL test environment", pytrace=False)
+ raw_port = values["ARES_TEST_POSTGRES_PORT"]
+ if not raw_port.isascii() or not raw_port.isdecimal():
+ pytest.fail("PostgreSQL test port is invalid", pytrace=False)
+ port = int(raw_port)
+ if str(port) != raw_port or not 1 <= port <= 65535:
+ pytest.fail("PostgreSQL test port is invalid", pytrace=False)
+ return _Config(
+ values["ARES_TEST_POSTGRES_HOST"],
+ port,
+ values["ARES_TEST_POSTGRES_USER"],
+ values["ARES_TEST_POSTGRES_DB"],
+ )
+
+
+def _url(config: _Config, database: str) -> str:
+ return URL.create(
+ "postgresql+asyncpg",
+ username=config.user,
+ host=config.host,
+ port=config.port,
+ database=database,
+ ).render_as_string(hide_password=False)
+
+
+@asynccontextmanager
+async def _database() -> AsyncIterator[str]:
+ config = _config()
+ try:
+ import asyncpg
+ except ImportError:
+ pytest.fail("PostgreSQL test driver is unavailable", pytrace=False)
+ name = f"ares_m2b_{uuid4().hex}"
+ if _SAFE_NAME.fullmatch(name) is None or name == config.database:
+ pytest.fail("Unsafe disposable database identity", pytrace=False)
+ admin = await asyncio.wait_for(
+ asyncpg.connect(
+ host=config.host,
+ port=config.port,
+ user=config.user,
+ database=config.database,
+ ),
+ timeout=20,
+ )
+ created = False
+ try:
+ created = True
+ await asyncio.wait_for(
+ admin.execute(f'CREATE DATABASE "{name}"'),
+ timeout=20,
+ )
+ yield _url(config, name)
+ finally:
+ if created:
+ await asyncio.wait_for(
+ admin.execute(
+ "SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
+ "WHERE datname=$1 AND pid <> pg_backend_pid()",
+ name,
+ ),
+ timeout=20,
+ )
+ await asyncio.wait_for(
+ admin.execute(f'DROP DATABASE IF EXISTS "{name}" WITH (FORCE)'),
+ timeout=20,
+ )
+ await asyncio.wait_for(admin.close(), timeout=20)
+
+
+async def _runtime(url: str, generation: int) -> None:
+ database = PostgresDatabase(url)
+ await database.connect()
+ await database.close()
+ if generation == 6:
+ engine = create_async_engine(url, poolclass=sa.pool.NullPool)
+ try:
+ async with engine.begin() as connection:
+ await connection.exec_driver_sql("DROP TABLE websocket_tickets")
+ finally:
+ await engine.dispose()
+
+
+async def _revision(url: str) -> str | None:
+ engine = create_async_engine(url, poolclass=sa.pool.NullPool)
+ try:
+ async with engine.connect() as connection:
+ relation = await connection.scalar(
+ sa.text(
+ "SELECT to_regclass(current_schema() || '.alembic_version')"
+ )
+ )
+ if relation is None:
+ return None
+ return await connection.scalar(
+ sa.text("SELECT version_num FROM alembic_version")
+ )
+ finally:
+ await engine.dispose()
+
+
+def test_postgres_adoption_config_absence_is_intentional_skip(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ for name in _ENV:
+ monkeypatch.delenv(name, raising=False)
+ with pytest.raises(pytest.skip.Exception):
+ _config()
+
+
+def test_postgres_adoption_partial_config_is_hard_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ for name in _ENV:
+ monkeypatch.delenv(name, raising=False)
+ monkeypatch.setenv("ARES_TEST_POSTGRES_HOST", "fixed-host")
+ with pytest.raises(pytest.fail.Exception):
+ _config()
+
+
+def test_postgres_adoption_invalid_port_is_hard_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ values = {
+ "ARES_TEST_POSTGRES_HOST": "fixed-host",
+ "ARES_TEST_POSTGRES_PORT": "not-a-port",
+ "ARES_TEST_POSTGRES_USER": "fixed-user",
+ "ARES_TEST_POSTGRES_DB": "fixed-db",
+ }
+ for name, value in values.items():
+ monkeypatch.setenv(name, value)
+ with pytest.raises(pytest.fail.Exception):
+ _config()
+
+
[email protected](("generation", "predecessor"), [(6, "0006"), (7, "0007")])
+async def test_real_postgres_generation_adopts_atomically(
+ monkeypatch: pytest.MonkeyPatch,
+ generation: int,
+ predecessor: str,
+) -> None:
+ async with _database() as url:
+ await _runtime(url, generation)
+ monkeypatch.setenv("ARES_DATABASE_URL", url)
+ verified = await adoption.execute("verify-adoption")
+ before = await _revision(url)
+ adopted = await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ after = await _revision(url)
+ managed = await adoption.execute("verify-managed")
+ reduced = (
+ verified.predecessor,
+ before,
+ adopted.exit_code,
+ after,
+ managed.diagnostic,
+ )
+ assert reduced == (
+ predecessor,
+ None,
+ AdoptionExit.OK,
+ "0008",
+ "ARES-M2B-ALREADY-MANAGED:0008",
+ )
+
+
+async def test_real_postgres_unknown_object_rejects_before_version_creation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async with _database() as url:
+ await _runtime(url, 7)
+ engine = create_async_engine(url, poolclass=sa.pool.NullPool)
+ try:
+ async with engine.begin() as connection:
+ await connection.exec_driver_sql("CREATE TABLE unexpected_object(value text)")
+ finally:
+ await engine.dispose()
+ monkeypatch.setenv("ARES_DATABASE_URL", url)
+ try:
+ await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ except AdoptionFailure as caught:
+ result_code = caught.exit_code
+ else:
+ result_code = AdoptionExit.OK
+ revision = await _revision(url)
+ assert (result_code, revision) == (AdoptionExit.UNRECOGNIZED, None)
+
+
+async def test_real_postgres_advisory_lock_has_one_owner(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async with _database() as url:
+ await _runtime(url, 7)
+ monkeypatch.setenv("ARES_DATABASE_URL", url)
+ engine = create_async_engine(url, poolclass=sa.pool.NullPool)
+ try:
+ async with engine.connect() as owner:
+ transaction = await owner.begin()
+ locked = await owner.scalar(
+ sa.text("SELECT pg_try_advisory_xact_lock(:key)"),
+ {"key": adoption._LOCK_KEY},
+ )
+ try:
+ await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ except AdoptionFailure as caught:
+ contender_code = caught.exit_code
+ else:
+ contender_code = AdoptionExit.OK
+ await transaction.rollback()
+ finally:
+ await engine.dispose()
+ winner = await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ assert (bool(locked), contender_code, winner.exit_code) == (
+ True,
+ AdoptionExit.OWNERSHIP,
+ AdoptionExit.OK,
+ )
+
+
+async def test_real_postgres_failure_after_upgrade_rolls_back(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async with _database() as url:
+ await _runtime(url, 7)
+ monkeypatch.setenv("ARES_DATABASE_URL", url)
+ original = adoption._run_canonical_adoption
+
+ def _fail_after_upgrade(connection: sa.Connection, predecessor: str) -> None:
+ original(connection, predecessor)
+ raise RuntimeError("fixed-injected-failure")
+
+ monkeypatch.setattr(adoption, "_run_canonical_adoption", _fail_after_upgrade)
+ with pytest.raises(RuntimeError, match="fixed-injected-failure"):
+ await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ rolled_back = await _revision(url)
+ monkeypatch.setattr(adoption, "_run_canonical_adoption", original)
+ retry = await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ assert (rolled_back, retry.exit_code) == (None, AdoptionExit.OK)
+
+
+async def test_real_postgres_unrelated_schema_is_preserved(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async with _database() as url:
+ await _runtime(url, 7)
+ engine = create_async_engine(url, poolclass=sa.pool.NullPool)
+ try:
+ async with engine.begin() as connection:
+ await connection.exec_driver_sql("CREATE SCHEMA unrelated_owner_data")
+ await connection.exec_driver_sql(
+ "CREATE TABLE unrelated_owner_data.marker(value integer)"
+ )
+ await connection.exec_driver_sql(
+ "INSERT INTO unrelated_owner_data.marker VALUES (7)"
+ )
+ finally:
+ await engine.dispose()
+ monkeypatch.setenv("ARES_DATABASE_URL", url)
+ result = await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ engine = create_async_engine(url, poolclass=sa.pool.NullPool)
+ try:
+ async with engine.connect() as connection:
+ preserved = await connection.scalar(
+ sa.text("SELECT count(*) FROM unrelated_owner_data.marker")
+ )
+ finally:
+ await engine.dispose()
+ assert (result.exit_code, preserved) == (AdoptionExit.OK, 1)
+
+
+async def test_real_postgres_adoption_rerun_is_verified_noop(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async with _database() as url:
+ await _runtime(url, 7)
+ monkeypatch.setenv("ARES_DATABASE_URL", url)
+ first = await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ second = await adoption.execute(
+ "adopt",
+ external_backup_confirmed=True,
+ )
+ revision = await _revision(url)
+ assert (first.exit_code, second.diagnostic, revision) == (
+ AdoptionExit.OK,
+ "ARES-M2B-ALREADY-MANAGED:0008",
+ "0008",
+ )
diff --git a/tests/unit/test_unversioned_database_adoption.py b/tests/unit/test_unversioned_database_adoption.py
new file mode 100644
index 0000000..1fd106c
--- /dev/null
+++ b/tests/unit/test_unversioned_database_adoption.py
@@ -0,0 +1,327 @@
+"""Authentic SQLite and packaging tests for explicit M2b adoption."""
+from __future__ import annotations
+
+import asyncio
+import os
+import sqlite3
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+import sqlalchemy as sa
+
+import ares.db.migrations.adoption as adoption_module
+from ares.db.database import AresDatabase
+from ares.db.migrations.adoption import (
+ SUPPORTED_GENERATIONS,
+ AdoptionExit,
+ AdoptionFailure,
+ AdoptionResult,
+ adopt_sqlite,
+ inspect_sqlite_database,
+ migration_config,
+ restore_sqlite,
+ sqlite_catalog_and_row_digest,
+ verify_managed_connection_from_path,
+)
+
+
+async def _create_runtime(path: Path, *, generation: int = 7) -> None:
+ database = AresDatabase(f"file:{path.resolve().as_posix()}?mode=rwc")
+ await database.connect()
+ await database.close()
+ if generation == 6:
+ connection = sqlite3.connect(path)
+ try:
+ connection.execute("PRAGMA foreign_keys=OFF")
+ connection.execute("DROP TABLE websocket_tickets")
+ connection.commit()
+ finally:
+ connection.close()
+
+
+def _catalog_digest(path: Path) -> bytes:
+ connection = sqlite3.connect(path)
+ try:
+ connection.execute("PRAGMA foreign_keys=ON")
+ return sqlite_catalog_and_row_digest(connection)
+ finally:
+ connection.close()
+
+
+def test_supported_generation_contract_is_frozen() -> None:
+ contract = tuple(
+ (item.dialect, item.generation, item.predecessor, item.has_websocket_tickets)
+ for item in SUPPORTED_GENERATIONS
+ )
+ assert contract == (
+ ("sqlite", 6, "0006", False),
+ ("sqlite", 7, "0007", True),
+ ("postgresql", 6, "0006", False),
+ ("postgresql", 7, "0007", True),
+ )
+
+
+def test_result_repr_and_equality_do_not_expose_fields() -> None:
+ result = AdoptionResult(AdoptionExit.OK, "ARES-M2B-EMPTY-DATABASE")
+ assert "ARES" not in repr(result)
+ assert result != AdoptionResult(AdoptionExit.OK, "ARES-M2B-EMPTY-DATABASE")
+
+
+def test_installed_resource_contract_and_linear_graph(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.chdir(Path.cwd().anchor)
+ with migration_config() as config:
+ script_location = config.get_main_option("script_location")
+ assert script_location
+ assert "alembic.ini" not in script_location
+
+
+def test_packaging_configuration_carries_one_canonical_tree() -> None:
+ project = Path(__file__).resolve().parents[2]
+ configuration = (project / "pyproject.toml").read_text(encoding="utf-8")
+ reduced = (
+ 'include = ["ares*", "migrations"]' in configuration,
+ 'migrations = ["script.py.mako", "versions/*.py"]' in configuration,
+ not (project / "ares" / "db" / "migrations" / "versions").exists(),
+ )
+ assert reduced == (True, True, True)
+
+
[email protected](("generation", "predecessor"), [(6, "0006"), (7, "0007")])
+async def test_sqlite_runtime_generation_is_classified(
+ tmp_path: Path,
+ generation: int,
+ predecessor: str,
+) -> None:
+ path = tmp_path / "runtime.db"
+ await _create_runtime(path, generation=generation)
+ result = inspect_sqlite_database(path)
+ assert result.predecessor == predecessor
+ assert result.diagnostic.endswith(predecessor)
+ connection = sqlite3.connect(path)
+ try:
+ version_count = connection.execute(
+ "SELECT count(*) FROM sqlite_schema WHERE name='alembic_version'"
+ ).fetchone()[0]
+ finally:
+ connection.close()
+ assert version_count == 0
+
+
+async def test_unknown_sqlite_catalog_is_rejected_without_mutation(
+ tmp_path: Path,
+) -> None:
+ path = tmp_path / "unknown.db"
+ await _create_runtime(path)
+ connection = sqlite3.connect(path)
+ try:
+ connection.execute("CREATE TABLE unexpected_object(value TEXT)")
+ connection.commit()
+ finally:
+ connection.close()
+ before = _catalog_digest(path)
+ with pytest.raises(AdoptionFailure) as caught:
+ inspect_sqlite_database(path)
+ rejected = (
+ caught.value.exit_code == AdoptionExit.UNRECOGNIZED,
+ _catalog_digest(path) == before,
+ )
+ assert rejected == (True, True)
+
+
[email protected]("generation", [6, 7])
+async def test_sqlite_adoption_uses_backup_and_normal_history(
+ tmp_path: Path,
+ generation: int,
+) -> None:
+ path = tmp_path / "source.db"
+ backup = tmp_path / "source.backup"
+ await _create_runtime(path, generation=generation)
+ before = _catalog_digest(path)
+ result = await asyncio.to_thread(adopt_sqlite, path, backup)
+ assert result.exit_code == AdoptionExit.OK
+ assert backup.is_file()
+ assert _catalog_digest(backup) == before
+ managed = verify_managed_connection_from_path(path)
+ assert managed.diagnostic == "ARES-M2B-ALREADY-MANAGED:0008"
+ connection = sqlite3.connect(path)
+ try:
+ revision = connection.execute(
+ "SELECT version_num FROM alembic_version"
+ ).fetchone()[0]
+ finally:
+ connection.close()
+ assert revision == "0008"
+
+
+async def test_sqlite_adoption_rerun_is_verified_noop(tmp_path: Path) -> None:
+ path = tmp_path / "source.db"
+ backup = tmp_path / "source.backup"
+ second_backup = tmp_path / "second.backup"
+ await _create_runtime(path)
+ first = await asyncio.to_thread(adopt_sqlite, path, backup)
+ before = _catalog_digest(path)
+ second = await asyncio.to_thread(adopt_sqlite, path, second_backup)
+ assert (
+ first.exit_code,
+ second.diagnostic,
+ _catalog_digest(path) == before,
+ second_backup.exists(),
+ ) == (
+ AdoptionExit.OK,
+ "ARES-M2B-ALREADY-MANAGED:0008",
+ True,
+ False,
+ )
+
+
+async def test_sqlite_post_upgrade_failure_rolls_back_exactly(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ path = tmp_path / "source.db"
+ backup = tmp_path / "source.backup"
+ await _create_runtime(path)
+ before = _catalog_digest(path)
+ original = adoption_module._run_canonical_adoption
+
+ def fail_after_upgrade(connection: sa.Connection, predecessor: str) -> None:
+ original(connection, predecessor)
+ raise RuntimeError("fixed-injected-failure")
+
+ monkeypatch.setattr(
+ adoption_module,
+ "_run_canonical_adoption",
+ fail_after_upgrade,
+ )
+ with pytest.raises(AdoptionFailure) as caught:
+ await asyncio.to_thread(adopt_sqlite, path, backup)
+ connection = sqlite3.connect(path)
+ try:
+ version_objects = connection.execute(
+ "SELECT count(*) FROM sqlite_schema WHERE name='alembic_version'"
+ ).fetchone()[0]
+ finally:
+ connection.close()
+ assert (
+ caught.value.exit_code,
+ _catalog_digest(path) == before,
+ backup.exists(),
+ version_objects,
+ ) == (AdoptionExit.ROLLED_BACK, True, True, 0)
+
+
+async def test_sqlite_backup_is_never_overwritten(tmp_path: Path) -> None:
+ path = tmp_path / "source.db"
+ backup = tmp_path / "source.backup"
+ await _create_runtime(path)
+ backup.write_bytes(b"occupied")
+ before = _catalog_digest(path)
+ with pytest.raises(AdoptionFailure) as caught:
+ await asyncio.to_thread(adopt_sqlite, path, backup)
+ fixed = (
+ caught.value.exit_code == AdoptionExit.BACKUP,
+ backup.read_bytes() == b"occupied",
+ _catalog_digest(path) == before,
+ )
+ assert fixed == (True, True, True)
+
+
+async def test_sqlite_restore_retains_verified_backup(tmp_path: Path) -> None:
+ path = tmp_path / "source.db"
+ backup = tmp_path / "source.backup"
+ await _create_runtime(path)
+ original = _catalog_digest(path)
+ source = sqlite3.connect(path)
+ target = sqlite3.connect(backup)
+ try:
+ source.backup(target)
+ finally:
+ target.close()
+ source.close()
+ connection = sqlite3.connect(path)
+ try:
+ connection.execute("CREATE TABLE drift(value TEXT)")
+ connection.commit()
+ finally:
+ connection.close()
+ result = await asyncio.to_thread(restore_sqlite, path, backup)
+ assert result.exit_code == AdoptionExit.OK
+ assert backup.exists()
+ assert _catalog_digest(path) == original
+
+
+async def test_startup_rejects_unversioned_without_alembic_side_effect(
+ tmp_path: Path,
+) -> None:
+ path = tmp_path / "runtime.db"
+ await _create_runtime(path)
+ before = _catalog_digest(path)
+ database = AresDatabase(path)
+ with pytest.raises(RuntimeError, match="adoption is required"):
+ await database.connect()
+ assert _catalog_digest(path) == before
+ assert database._conn is None
+
+
+async def test_startup_rejects_noncanonical_fake_0008(tmp_path: Path) -> None:
+ path = tmp_path / "runtime.db"
+ await _create_runtime(path)
+ connection = sqlite3.connect(path)
+ try:
+ connection.execute(
+ "CREATE TABLE alembic_version ("
+ "version_num VARCHAR(32) NOT NULL, "
+ "CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num))"
+ )
+ connection.execute("INSERT INTO alembic_version VALUES ('0008')")
+ connection.commit()
+ finally:
+ connection.close()
+ before = _catalog_digest(path)
+ database = AresDatabase(path)
+ with pytest.raises(RuntimeError, match="Invalid managed"):
+ await database.connect()
+ assert _catalog_digest(path) == before
+
+
+def test_cli_help_has_no_repository_dependency(tmp_path: Path) -> None:
+ environment = dict(os.environ)
+ environment["PYTHONPYCACHEPREFIX"] = os.fspath(tmp_path / "pycache")
+ environment["PYTHONPATH"] = os.fspath(Path(__file__).resolve().parents[2])
+ completed = subprocess.run(
+ [sys.executable, "-m", "ares.db.migrations", "--help"],
+ cwd=tmp_path,
+ env=environment,
+ capture_output=True,
+ text=True,
+ timeout=30,
+ check=False,
+ )
+ reduced = (
+ completed.returncode,
+ "verify-adoption" in completed.stdout,
+ "ARES-M2B-E02" not in completed.stdout,
+ )
+ assert reduced == (0, True, True)
+
+
+def test_caller_owned_connection_remains_open(tmp_path: Path) -> None:
+ engine = sa.create_engine(f"sqlite:///{tmp_path / 'empty.db'}")
+ try:
+ with engine.connect() as connection:
+ with migration_config(connection) as config:
+ assert config.attributes["connection"] is connection
+ assert connection.scalar(sa.text("SELECT 1")) == 1
+ finally:
+ engine.dispose()