Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(storage): expire verified migration backups after three days (#2252)

Call White072f8202c6dd
diff --git a/core/controller.py b/core/controller.py
index 35c7c1095..aeb7b3317 100644
--- a/core/controller.py
+++ b/core/controller.py
@@ -2057,6 +2057,7 @@ class Controller:
from storage import agent_events_retention
from storage.db import get_cached_sqlite_engine
+ self._run_sqlite_backup_expiry_pass(cancel_event)
# Skill privacy retention remains active when collection/tool retention
# is disabled; both policies share this bounded maintenance worker.
engine = get_cached_sqlite_engine()
@@ -2071,6 +2072,33 @@ class Controller:
cancel_event=cancel_event,
)
+ def _run_sqlite_backup_expiry_pass(self, cancel_event=None) -> dict:
+ import os
+
+ from config import paths
+ from storage.backups import expire_sqlite_migration_backups
+ from vibe.runtime import (
+ confirm_pending_sqlite_backup_readiness,
+ current_process_owns_service_instance,
+ service_instance_started,
+ )
+
+ if cancel_event is not None and cancel_event.is_set():
+ return {"status": "cancelled", "removed": []}
+ if not current_process_owns_service_instance() or not service_instance_started(os.getpid()):
+ return {"status": "not_owner", "removed": []}
+ try:
+ confirm_pending_sqlite_backup_readiness()
+ result = expire_sqlite_migration_backups(
+ paths.get_sqlite_state_path(), cancel_event=cancel_event
+ )
+ if result["status"] == "unconfirmed":
+ logger.info("SQLite backup expiry deferred: no verified ready migration lifecycle")
+ return result
+ except Exception:
+ logger.warning("SQLite backup expiry deferred; retaining rollback files", exc_info=True)
+ return {"status": "deferred", "removed": []}
+
async def _join_trace_retention_future(self) -> None:
"""Wait for the worker's current batch to observe cooperative stop."""
diff --git a/docs/CLI.md b/docs/CLI.md
index 400e09f1a..50056bcf6 100644
--- a/docs/CLI.md
+++ b/docs/CLI.md
@@ -677,6 +677,27 @@ The web UI (`http://127.0.0.1:5123`) provides the same controls:
| `~/.vibe_remote/logs/vibe_remote.log` | Application logs |
| `~/.vibe_remote/logs/opencode_server.json` | OpenCode server PID file |
+### Migration Backup Lifetime
+
+Automatic SQLite migration backups are temporary upgrade rollback points, not
+long-term archives. A backup becomes eligible for deletion 72 hours after its
+migration commits, database validation passes, and the core service reports
+readiness. Restarting the service does not extend this deadline. The last
+eligible copy can be deleted too.
+
+The core service checks on startup and hourly, including when trace retention
+is disabled. An offline installation catches up after startup. The existing
+rollback-position count limit still applies independently, so older copies can
+leave that window before 72 hours.
+
+Failed, interrupted, restored, or unconfirmed migration attempts are not
+age-expired. Legacy backups without recorded success evidence retain their
+existing count policy; their creation dates are not used to invent an expiry
+deadline. JSON import backups, manual archives, and unrecognized files are not
+covered by this age policy. Restoring a SQLite backup revokes the old expiry
+evidence before replacing the live database. Keep a separate manual archive
+when recovery must remain possible beyond the automatic rollback window.
+
## Environment Variables
| Variable | Description |
diff --git a/docs/plans/sqlite-backup-expiry.md b/docs/plans/sqlite-backup-expiry.md
new file mode 100644
index 000000000..c2c266e8f
--- /dev/null
+++ b/docs/plans/sqlite-backup-expiry.md
@@ -0,0 +1,117 @@
+# SQLite Migration Backup Expiry
+
+## Contract
+
+Managed SQLite migration backups are short-lived upgrade rollback points, not
+permanent user archives. After their own migration has committed, database
+validation has passed, and the owning core service has become ready, they expire
+after 72 hours. The final eligible backup may expire; no minimum copy count
+overrides expiry.
+
+- Record the successful lifecycle durably and bind it to the actual migration
+ attempt and database. A package-install exit code, matching revision label,
+ file mtime, or later unrelated service startup is not proof of that attempt.
+- Readiness starts the clock once. Restarts must not renew it. Failed,
+ interrupted, unverified, or restored attempts do not qualify for age expiry.
+- Retain the existing bounded rollback-position policy independently. Older
+ backups can be evicted by that bound before 72 hours; this is a maximum normal
+ retention age, not a promise that every copy survives for three days.
+- Only recognized Avibe SQLite migration backups qualify. JSON import backups,
+ hand-made files, symlinks, malformed manifests, and legacy backups without
+ success evidence keep their existing policies.
+- Do not manufacture completion evidence for old backups. Deploying this change
+ must not bulk-expire existing copies based on their creation dates.
+- A restore invalidates prior expiry authority before changing the database.
+ Preserve displaced database/WAL recovery evidence and serialize migration,
+ restore, and expiry against the same database lock.
+- A failed or unconfirmed migration must not let an old readiness signal
+ authorize expiry of the rollback evidence needed by that attempt.
+- Reuse the controller's bounded maintenance worker: inspect on startup and
+ hourly, independently of optional trace-retention/Skill settings. Offline
+ machines catch up when started. Do not introduce a Watch or scheduled Task.
+- Expiry is optional maintenance: contention or failure defers deletion and is
+ logged, never makes a healthy service fail startup. Log actual removal and
+ reasons for deferral without exposing database contents.
+
+## Implementation Boundaries
+
+The backup module owns lifecycle metadata, candidate admission, and deletion.
+The migration/import path produces completion evidence only after its actual
+work and validation succeed. The established service-readiness owner confirms
+readiness; the existing maintenance worker only consumes that evidence.
+
+Use additive, backward-compatible backup metadata and existing filesystem/lock
+helpers. No new dependency, UI control, operating-system scheduler, production
+cleanup, service restart, or deployment is part of this change.
+
+### Lifecycle Evidence
+
+The database-owned sidecar and the backup manifest share one attempt ID,
+canonical database path, file identity, target revisions, and lifecycle phase.
+Before a migration can mutate SQLite, it durably publishes a blocking
+`migrating` sidecar. Only successful Alembic completion and a SQLite check advance
+it to `migrated`. The importer records `validating` before data work and
+`validated` only after commit and validation. Its report carries the exact
+attempt ID in the existing process-local ensured-state cache. The actual
+service-readiness owner consumes only an attempt validated by that process,
+checks the live and packaged schema heads, and records `ready` once. It never
+discovers a readiness attempt by selecting the latest on-disk receipt.
+
+The service captures the exact attempt and its original readiness time. A
+try-lock miss retains this pending event for the existing maintenance worker
+to retry, without renewing the clock or adopting another attempt. Releasing
+the service-instance lock drops the pending event. If the process dies before
+confirmation, the next startup must perform validation and readiness again;
+there is no inferred success or old timestamp to recover.
+
+Manifest and sidecar updates use private temporary files, atomic replacement,
+and file/directory synchronization. Disagreement after an interrupted write
+does not grant expiry authority. Optional completion metadata failures keep
+the service usable and retain backups; failure to establish the initial
+blocking record aborts before SQL mutation.
+
+The expiry worker takes the existing migration lock without waiting. It checks
+the latest lifecycle and live revisions before considering individual ready
+manifests. A replacement database cannot inherit the old file's authority.
+Unknown directory contents, including databases displaced during restore,
+prevent age deletion. Restore writes its blocking state before touching live
+data and revokes all recorded ready attempts.
+
+Relocated/symlinked backup roots remain usable by the existing backup producer,
+but receive only a blocking sidecar and no age-expiry receipt. This preserves
+upgrades on installations that moved their backup directory to another disk.
+
+## Validation
+
+- Real SQLite migration to readiness to expiry, including unchanged restarts.
+- Strict 72-hour boundary, offset-aware timestamps, future/invalid timestamps.
+- Last-copy expiry and interaction with the existing count bound.
+- Migration/import failure, crash-before-confirmation, unrelated startup,
+ repeated partial failures, and malformed/legacy metadata fail closed.
+- Restore invalidation, displaced data preservation, and lock contention.
+- Manual files, JSON backups, symlinks, and the live database are untouched.
+- Startup/hourly maintenance continues with trace retention disabled and does
+ not block readiness or outlive the controller's worker shutdown.
+- A real lock miss at readiness is retried by controller maintenance using the
+ same attempt ID and original timestamp; another process's validation or a
+ newer attempt with identical revisions cannot borrow that readiness.
+- Hermetic state roots, focused regression tests, changed-file lint, exact-head
+ Codex review and CI. No local production mutation or automatic merge.
+
+Local validation: 326 tests and four subtests passed across the focused expiry,
+backup, migration/import, startup, service-lock, controller readiness/retention,
+and development-state guard suites. Two Windows byte-range lock tests were
+skipped on macOS. Changed-file Ruff and whitespace checks passed. The read-only
+architecture review's exact-startup identity and deferred-readiness findings
+are covered by the process-local attempt report and real contention/maintenance
+consumer tests. Required remote CI and exact-head review are still separate
+delivery gates.
+
+## Progress
+
+- [x] Inspect backup, migration/import, readiness, restore, and maintenance owners.
+- [x] Implement durable lifecycle admission and 72-hour expiry.
+- [x] Wire producers, readiness, restore protection, and maintenance.
+- [x] Add focused regression and lifecycle integration coverage.
+- [x] Complete focused local validation and independent architecture feedback.
+- [ ] Publish a non-draft PR and satisfy exact-head review/CI gates.
diff --git a/storage/backups.py b/storage/backups.py
index b1d7ad016..855ea82a2 100644
--- a/storage/backups.py
+++ b/storage/backups.py
@@ -7,11 +7,15 @@ import os
import re
import shutil
import sqlite3
+import tempfile
+import uuid
from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Iterable
+from storage.lock import MigrationFileLock, MigrationLockTimeout, migration_lock_path_for
+
logger = logging.getLogger(__name__)
@@ -21,6 +25,7 @@ logger = logging.getLogger(__name__)
JSON_STATE_BACKUP_RETENTION = 3
SQLITE_BACKUP_RETENTION = 2
BACKUP_MANIFEST_VERSION = 1
+SQLITE_BACKUP_MAX_AGE = timedelta(hours=72)
#: The name a restore gives the database it takes out of service, inside the
#: backup directory it restored from. That directory is the record of "we rolled
@@ -133,7 +138,7 @@ def _read_manifest(path: Path) -> dict | None:
return None
try:
payload = json.loads(path.read_text(encoding="utf-8"))
- except (OSError, json.JSONDecodeError):
+ except (OSError, UnicodeError, json.JSONDecodeError):
return None
return payload if isinstance(payload, dict) else None
@@ -524,6 +529,19 @@ def create_sqlite_migration_backup(
backups_dir: Path | None = None,
to_revisions: Iterable[str] = (),
now: datetime | None = None,
+) -> Path:
+ with MigrationFileLock(migration_lock_path_for(db_path), timeout_seconds=None):
+ return _create_sqlite_migration_backup_locked(
+ db_path, backups_dir=backups_dir, to_revisions=to_revisions, now=now
+ )
+
+
+def _create_sqlite_migration_backup_locked(
+ db_path: Path,
+ *,
+ backups_dir: Path | None = None,
+ to_revisions: Iterable[str] = (),
+ now: datetime | None = None,
) -> Path:
"""Hold a rollback point for the database as it stands, in a bounded window.
@@ -632,6 +650,295 @@ def create_sqlite_migration_backup(
return backup_dir
+def _lifecycle_path(db_path: Path) -> Path:
+ return db_path.with_name(f".{db_path.name}.backup-lifecycle.json")
+
+
+def _write_backup_metadata(path: Path, payload: dict) -> None:
+ """Publish one private receipt durably, without exposing a partial JSON file."""
+ if path.is_symlink():
+ raise OSError(f"Refusing symlink backup metadata: {path}")
+ descriptor, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
+ temporary = Path(name)
+ try:
+ with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
+ json.dump(payload, stream, indent=2)
+ stream.write("\n")
+ stream.flush()
+ os.fsync(stream.fileno())
+ temporary.replace(path)
+ _fsync_directory(path.parent)
+ finally:
+ temporary.unlink(missing_ok=True)
+
+
+def _database_identity(db_path: Path) -> list[int]:
+ info = db_path.stat()
+ return [info.st_dev, info.st_ino]
+
+
+def _valid_lifecycle_shape(record: object, db_path: Path) -> bool:
+ return isinstance(record, dict) and (
+ type(record.get("schema_version")) is int
+ and record["schema_version"] == 1
+ and record.get("database") == str(db_path)
+ and isinstance(record.get("attempt_id"), str)
+ and re.fullmatch(r"[0-9a-f]{32}", record["attempt_id"]) is not None
+ and isinstance(record.get("backup_name"), str)
+ and _SQLITE_BACKUP_RE.fullmatch(record["backup_name"]) is not None
+ and isinstance(record.get("to_revisions"), list)
+ and bool(record["to_revisions"])
+ and all(isinstance(value, str) and value for value in record["to_revisions"])
+ and isinstance(record.get("database_identity"), list)
+ and len(record["database_identity"]) == 2
+ and all(type(value) is int for value in record["database_identity"])
+ and isinstance(record.get("phase"), str)
+ and record.get("phase") in {"migrating", "migrated", "validating", "validated", "ready", "restoring", "restored"}
+ )
+
+
+def _lifecycle_record(db_path: Path) -> dict | None:
+ record = _read_manifest(_lifecycle_path(db_path))
+ if not (
+ _valid_lifecycle_shape(record, db_path)
+ and record["database_identity"] == _database_identity(db_path)
+ ):
+ return None
+ return record
+
+
+def _attempt_manifest(db_path: Path, record: dict) -> tuple[Path, dict] | None:
+ root = db_path.parent / "backups"
+ if root.is_symlink():
+ return None
+ backup = root / record["backup_name"]
+ candidate = _directory_candidate(backup)
+ manifest = _read_manifest(backup / "manifest.json")
+ if (
+ candidate is None
+ or candidate.kind != "sqlite"
+ or manifest is None
+ or manifest.get("expiry_lifecycle") != record
+ or manifest.get("to_revisions") != record["to_revisions"]
+ ):
+ return None
+ return backup / "manifest.json", manifest
+
+
+def _advance_backup_lifecycle(db_path: Path, record: dict, *, phase: str, **fields) -> None:
+ admitted = _attempt_manifest(db_path, record)
+ if admitted is None:
+ raise OSError("Migration backup lifecycle no longer matches its manifest")
+ manifest_path, manifest = admitted
+ updated = {**record, **fields, "phase": phase}
+ manifest["expiry_lifecycle"] = updated
+ # A crash between these writes leaves disagreement, never deletion authority.
+ _write_backup_metadata(manifest_path, manifest)
+ _write_backup_metadata(_lifecycle_path(db_path), updated)
+
+
+def begin_sqlite_backup_migration(db_path: Path, backup_dir: Path) -> str | None:
+ """Invalidate prior expiry authority before this attempt can change the DB."""
+ target = db_path.expanduser().resolve()
+ with MigrationFileLock(migration_lock_path_for(target), timeout_seconds=None):
+ candidate = _directory_candidate(backup_dir)
+ manifest_path = backup_dir / "manifest.json"
+ manifest = _read_manifest(manifest_path)
+ if candidate is None or candidate.kind != "sqlite" or manifest is None:
+ raise ValueError("Migration backup is not a recognized SQLite rollback point")
+ record = {
+ "schema_version": 1,
+ "attempt_id": uuid.uuid4().hex,
+ "database": str(target),
+ "database_identity": _database_identity(target),
+ "backup_name": backup_dir.name,
+ "to_revisions": manifest["to_revisions"],
+ "phase": "migrating",
+ }
+ # Publish the blocking record first; an incomplete attempt must stop
+ # expiry even when the subsequent manifest write or migration fails.
+ _write_backup_metadata(_lifecycle_path(target), record)
+ root = target.parent / "backups"
+ if backup_dir.parent != root or root.is_symlink():
+ logger.info("SQLite backup expiry disabled for relocated backup directory %s", backup_dir.parent)
+ return None
+ manifest["expiry_lifecycle"] = record
+ _write_backup_metadata(manifest_path, manifest)
+ return record["attempt_id"]
+
+
+def _validate_migrated_database(db_path: Path, revisions: Iterable[str]) -> None:
+ with sqlite3.connect(f"{db_path.as_uri()}?mode=ro", uri=True) as connection:
+ if _stamped_revisions(connection) != _normalized_position(revisions):
+ raise sqlite3.DatabaseError("Migration did not reach its recorded target revisions")
+ if connection.execute("PRAGMA quick_check").fetchone() != ("ok",):
+ raise sqlite3.DatabaseError("Migrated SQLite database failed quick_check")
+
+
+def complete_sqlite_backup_migration(
+ db_path: Path, attempt_id: str, *, now: datetime | None = None
+) -> None:
+ target = db_path.expanduser().resolve()
+ with MigrationFileLock(migration_lock_path_for(target), timeout_seconds=None):
+ record = _lifecycle_record(target)
+ if record is None or record["attempt_id"] != attempt_id or record.get("phase") != "migrating":
+ raise OSError("SQLite migration completion lost its exact backup attempt")
+ _validate_migrated_database(target, record["to_revisions"])
+ _advance_backup_lifecycle(
+ target,
+ record,
+ phase="migrated",
+ migration_completed_at=(now or datetime.now(timezone.utc)).isoformat(),
+ )
+
+
+def begin_sqlite_backup_validation(db_path: Path) -> str | None:
+ target = db_path.expanduser().resolve()
+ with MigrationFileLock(migration_lock_path_for(target), timeout_seconds=None):
+ record = _lifecycle_record(target)
+ if record is None or record.get("phase") not in {"migrated", "validating", "validated"}:
+ return None
+ _advance_backup_lifecycle(target, record, phase="validating")
+ return record["attempt_id"]
+
+
+def complete_sqlite_backup_validation(
+ db_path: Path, attempt_id: str | None, *, now: datetime | None = None
+) -> None:
+ if attempt_id is None:
+ return
+ target = db_path.expanduser().resolve()
+ with MigrationFileLock(migration_lock_path_for(target), timeout_seconds=None):
+ record = _lifecycle_record(target)
+ if record is None or record["attempt_id"] != attempt_id or record.get("phase") != "validating":
+ raise OSError("SQLite state validation lost its exact backup attempt")
+ _validate_migrated_database(target, record["to_revisions"])
+ _advance_backup_lifecycle(
+ target,
+ record,
+ phase="validated",
+ validated_at=(now or datetime.now(timezone.utc)).isoformat(),
+ )
+
+
+def confirm_sqlite_backup_readiness(
+ db_path: Path, *, attempt_id: str, expected_revisions: Iterable[str], now: datetime | None = None
+) -> bool:
+ """Only the actual service-readiness owner calls this, not the hourly sweep."""
+ target = db_path.expanduser().resolve()
+ if not target.is_file() or not _lifecycle_path(target).is_file():
+ return False
+ with MigrationFileLock(migration_lock_path_for(target), timeout_seconds=0):
+ record = _lifecycle_record(target)
+ if record is None or record["attempt_id"] != attempt_id or record.get("phase") != "validated":
+ return False
+ if _normalized_position(expected_revisions) != _normalized_position(record["to_revisions"]):
+ return False
+ with sqlite3.connect(f"{target.as_uri()}?mode=ro", uri=True) as connection:
+ if _stamped_revisions(connection) != _normalized_position(record["to_revisions"]):
+ return False
+ _advance_backup_lifecycle(
+ target,
+ record,
+ phase="ready",
+ ready_at=(now or datetime.now(timezone.utc)).isoformat(),
+ )
+ return True
+
+
+def _lifecycle_time(value: object) -> datetime | None:
+ if not isinstance(value, str):
+ return None
+ try:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+ return parsed.astimezone(timezone.utc) if parsed.tzinfo is not None else None
+ except (ValueError, OverflowError):
+ return None
+
+
+def expire_sqlite_migration_backups(
+ db_path: Path, *, now: datetime | None = None, cancel_event=None
+) -> dict:
+ """Remove only proven successful copies; a pending/failed attempt fences all."""
+ target = db_path.expanduser().resolve()
+ root = target.parent / "backups"
+ current_time = now or datetime.now(timezone.utc)
+ if current_time.tzinfo is None:
+ raise ValueError("Backup expiry requires an offset-aware timestamp")
+ if not target.is_file() or not root.is_dir() or root.is_symlink():
+ return {"status": "not_due", "removed": []}
+ removed: list[str] = []
+ try:
+ with MigrationFileLock(migration_lock_path_for(target), timeout_seconds=0):
+ record = _lifecycle_record(target)
+ if record is None or record.get("phase") != "ready":
+ return {"status": "unconfirmed", "removed": []}
+ with sqlite3.connect(f"{target.as_uri()}?mode=ro", uri=True) as connection:
+ if _stamped_revisions(connection) != _normalized_position(record["to_revisions"]):
+ return {"status": "unconfirmed", "removed": []}
+ for candidate in _managed_candidates(root):
+ if cancel_event is not None and cancel_event.is_set():
+ return {"status": "cancelled", "removed": removed}
+ if candidate.kind != "sqlite" or not candidate.root.is_dir():
+ continue
+ manifest = _read_manifest(candidate.root / "manifest.json") or {}
+ lifecycle = manifest.get("expiry_lifecycle")
+ if not _valid_lifecycle_shape(lifecycle, target) or not (
+ lifecycle.get("phase") == "ready"
+ and lifecycle.get("database_identity") == record["database_identity"]
+ and lifecycle.get("backup_name") == candidate.root.name
+ and lifecycle.get("to_revisions") == manifest.get("to_revisions")
+ ):
+ continue
+ completed = _lifecycle_time(lifecycle.get("migration_completed_at"))
+ validated = _lifecycle_time(lifecycle.get("validated_at"))
+ ready = _lifecycle_time(lifecycle.get("ready_at"))
+ if not (
+ completed is not None
+ and validated is not None
+ and ready is not None
+ and completed <= validated <= ready <= current_time
+ and current_time - ready >= SQLITE_BACKUP_MAX_AGE
+ ):
+ continue
+ # Restored/displaced data and user-added files are not temporary
+ # upgrade copies. Never remove them through the new age policy.
+ if {entry.name for entry in candidate.root.iterdir()} != {"manifest.json", "vibe.sqlite"}:
+ continue
+ logical_bytes = (candidate.root / "vibe.sqlite").stat().st_size
+ if _remove_candidate(candidate):
+ removed.append(candidate.root.name)
+ logger.info(
+ "Expired SQLite migration backup %s after 72 hours (logical bytes=%d)",
+ candidate.root.name,
+ logical_bytes,
+ )
+ except MigrationLockTimeout:
+ logger.info("SQLite backup expiry deferred: migration or restore owns the database lock")
+ return {"status": "busy", "removed": []}
+ return {"status": "ok" if removed else "not_due", "removed": removed}
+
+
+def _invalidate_backup_expiry_for_restore(db_path: Path) -> None:
+ lifecycle_path = _lifecycle_path(db_path)
+ if not lifecycle_path.exists() and not lifecycle_path.is_symlink():
+ return
+ record = _read_manifest(lifecycle_path) or {}
+ _write_backup_metadata(lifecycle_path, {**record, "phase": "restoring"})
+ root = db_path.parent / "backups"
+ if root.is_symlink():
+ return
+ for candidate in _managed_candidates(root):
+ if candidate.kind != "sqlite" or not candidate.root.is_dir():
+ continue
+ manifest_path = candidate.root / "manifest.json"
+ manifest = _read_manifest(manifest_path) or {}
+ lifecycle = manifest.get("expiry_lifecycle")
+ if isinstance(lifecycle, dict) and lifecycle.get("database") == str(db_path):
+ manifest["expiry_lifecycle"] = {**lifecycle, "phase": "restored"}
+ _write_backup_metadata(manifest_path, manifest)
+
+
def _duplicate_file(source: Path, destination: Path) -> None:
"""Copy `source` to `destination` without a window where only one copy exists.
@@ -822,6 +1129,14 @@ def _swap_live_database(db_path: Path, replacement: Path, *, into: Path) -> Path
def restore_sqlite_backup(backup_dir: Path, db_path: Path) -> Path | None:
+ target = db_path.expanduser().resolve()
+ target.parent.mkdir(parents=True, exist_ok=True)
+ with MigrationFileLock(migration_lock_path_for(target), timeout_seconds=None):
+ _invalidate_backup_expiry_for_restore(target)
+ return _restore_sqlite_backup_locked(backup_dir, target)
+
+
+def _restore_sqlite_backup_locked(backup_dir: Path, db_path: Path) -> Path | None:
"""Put a rollback point back into service, destroying nothing.
A restore is a swap, never an overwrite. The database it takes out of service
diff --git a/storage/importer.py b/storage/importer.py
index 4efe145c3..633c36cf1 100644
--- a/storage/importer.py
+++ b/storage/importer.py
@@ -5,7 +5,7 @@ import logging
import shutil
import tempfile
import threading
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
@@ -23,7 +23,13 @@ from config.v2_sessions import (
migrate_session_state_mappings,
)
from config.v2_settings import SettingsState, load_settings_state_from_json
-from storage.backups import BACKUP_MANIFEST_VERSION, next_backup_sequence, prune_state_backups
+from storage.backups import (
+ BACKUP_MANIFEST_VERSION,
+ begin_sqlite_backup_validation,
+ complete_sqlite_backup_validation,
+ next_backup_sequence,
+ prune_state_backups,
+)
from storage.db import create_sqlite_engine
from storage.lock import MigrationFileLock, migration_lock_path_for
from storage.migrations import guard_source_checkout_default_state_migration, run_migrations
@@ -53,12 +59,25 @@ class MigrationImportReport:
imported: bool
backup_path: Path | None = None
counts: dict[str, int] = field(default_factory=dict)
+ backup_attempt_id: str | None = None
_ensured_lock = threading.Lock()
_ensured_targets: dict[tuple[Path, Path], MigrationImportReport] = {}
+def validated_sqlite_backup_attempt(db_path: Path) -> str | None:
+ """The exact backup attempt this process validated, never one guessed from disk."""
+ target = db_path.expanduser().resolve()
+ with _ensured_lock:
+ attempts = {
+ report.backup_attempt_id
+ for (database, _state_dir), report in _ensured_targets.items()
+ if database == target and report.backup_attempt_id is not None
+ }
+ return attempts.pop() if len(attempts) == 1 else None
+
+
def reset_ensured_sqlite_state() -> None:
"""Forget which targets this process already ensured.
@@ -109,6 +128,11 @@ def ensure_sqlite_state(
# it in between.
with MigrationFileLock(migration_lock_path_for(target_db), timeout_seconds=None):
run_migrations(target_db, prune_backups_after_upgrade=False)
+ try:
+ backup_attempt = begin_sqlite_backup_validation(target_db)
+ except OSError:
+ logger.warning("Could not track SQLite backup validation; retaining backups", exc_info=True)
+ backup_attempt = None
engine = create_sqlite_engine(target_db)
report: MigrationImportReport | None = None
try:
@@ -169,6 +193,12 @@ def ensure_sqlite_state(
engine.dispose()
if report is None:
raise RuntimeError("SQLite state initialization completed without a report")
+ try:
+ complete_sqlite_backup_validation(target_db, backup_attempt)
+ except OSError:
+ logger.warning("SQLite state validated without expiry evidence; retaining backups", exc_info=True)
+ else:
+ report = replace(report, backup_attempt_id=backup_attempt)
prune_state_backups(target_state_dir / "backups")
with _ensured_lock:
_ensured_targets[ensured_key] = report
diff --git a/storage/migrations.py b/storage/migrations.py
index 14e2bc6f7..b9f09234c 100644
--- a/storage/migrations.py
+++ b/storage/migrations.py
@@ -13,7 +13,12 @@ from alembic.config import Config
from alembic.script import ScriptDirectory
from config import paths
-from storage.backups import create_sqlite_migration_backup, prune_state_backups
+from storage.backups import (
+ begin_sqlite_backup_migration,
+ complete_sqlite_backup_migration,
+ create_sqlite_migration_backup,
+ prune_state_backups,
+)
from storage.db import create_sqlite_engine, sqlite_url
from storage.lock import MigrationFileLock, migration_lock_path_for
@@ -297,14 +302,21 @@ def _run_migrations_locked(
) -> None:
cfg = alembic_config(target_db)
backup_revisions = _migration_backup_revisions(target_db, cfg, revision)
+ backup_attempt: str | None = None
if backup_revisions is not None:
_current_revisions, target_revisions = backup_revisions
backup_path = create_sqlite_migration_backup(target_db, to_revisions=target_revisions)
+ backup_attempt = begin_sqlite_backup_migration(target_db, backup_path)
logger.info("Pre-migration SQLite rollback point at %s", backup_path)
_reset_unreleased_initial_schema_drift(target_db)
_repair_unreleased_head_schema_drift(target_db)
_stamp_existing_initial_schema(target_db, cfg)
command.upgrade(cfg, revision)
+ if backup_attempt is not None:
+ try:
+ complete_sqlite_backup_migration(target_db, backup_attempt)
+ except OSError:
+ logger.warning("SQLite migration completed without expiry evidence; retaining backups", exc_info=True)
if prune_backups_after_upgrade:
prune_state_backups(target_db.parent / "backups", json_retention=None)
diff --git a/tests/test_controller_trace_retention.py b/tests/test_controller_trace_retention.py
index 90907a2c3..9dd3812a5 100644
--- a/tests/test_controller_trace_retention.py
+++ b/tests/test_controller_trace_retention.py
@@ -336,3 +336,50 @@ def test_retention_loop_shutdown_joins_executor(monkeypatch) -> None:
asyncio.run(_main())
assert controller.trace_retention_task is None
assert controller._trace_retention_executor is None
+
+
+def test_backup_expiry_is_independent_of_trace_retention_opt_out(monkeypatch) -> None:
+ controller = Controller.__new__(Controller)
+ calls = []
+ monkeypatch.setattr(Controller, "_agent_events_retention_config", lambda self: None)
+ monkeypatch.setattr("storage.db.get_cached_sqlite_engine", lambda: object())
+ monkeypatch.setattr(real_module, "run_skill_retention", lambda *args, **kwargs: {})
+ monkeypatch.setattr("vibe.runtime.current_process_owns_service_instance", lambda: True)
+ monkeypatch.setattr("vibe.runtime.service_instance_started", lambda pid: True)
+ monkeypatch.setattr(
+ "storage.backups.expire_sqlite_migration_backups",
+ lambda path, **kwargs: calls.append(path) or {"status": "not_due", "removed": []},
+ )
+
+ assert controller._run_agent_events_retention_pass() == {"status": "disabled"}
+ assert len(calls) == 1
+
+
+def test_backup_expiry_failure_does_not_block_other_maintenance(monkeypatch) -> None:
+ controller = Controller.__new__(Controller)
+ calls = []
+ monkeypatch.setattr(Controller, "_agent_events_retention_config", lambda self: None)
+ monkeypatch.setattr("storage.db.get_cached_sqlite_engine", lambda: object())
+ monkeypatch.setattr(real_module, "run_skill_retention", lambda *args, **kwargs: calls.append("skill"))
+ monkeypatch.setattr("vibe.runtime.current_process_owns_service_instance", lambda: True)
+ monkeypatch.setattr("vibe.runtime.service_instance_started", lambda pid: True)
+
+ def fail_expiry(*args, **kwargs):
+ raise OSError("unreadable backup metadata")
+
+ monkeypatch.setattr("storage.backups.expire_sqlite_migration_backups", fail_expiry)
+ assert controller._run_agent_events_retention_pass() == {"status": "disabled"}
+ assert calls == ["skill"]
+
+
[email protected](("owner", "ready"), [(False, True), (True, False)])
+def test_non_owner_or_unready_controller_does_not_expire_backups(monkeypatch, owner, ready) -> None:
+ controller = Controller.__new__(Controller)
+ monkeypatch.setattr("vibe.runtime.current_process_owns_service_instance", lambda: owner)
+ monkeypatch.setattr("vibe.runtime.service_instance_started", lambda pid: ready)
+
+ def unexpected(*args, **kwargs):
+ pytest.fail("An unready or non-owner controller must not reach expiry")
+
+ monkeypatch.setattr("storage.backups.expire_sqlite_migration_backups", unexpected)
+ assert controller._run_sqlite_backup_expiry_pass()["status"] == "not_owner"
diff --git a/tests/test_sqlite_backup_expiry.py b/tests/test_sqlite_backup_expiry.py
new file mode 100644
index 000000000..9362d5a0d
--- /dev/null
+++ b/tests/test_sqlite_backup_expiry.py
@@ -0,0 +1,597 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pytest
+
+from storage import backups
+from storage.importer import ensure_sqlite_state
+from storage.lock import MigrationFileLock, migration_lock_path_for
+from storage.migrations import run_migrations
+
+
+START = datetime(2026, 9, 28, 12, 0, tzinfo=timezone.utc)
+READY = START + timedelta(seconds=20)
+EXPIRES = READY + timedelta(hours=72)
+
+
+def _database(tmp_path: Path) -> Path:
+ target = tmp_path / "donn\u00e9es" / "vibe.sqlite"
+ target.parent.mkdir()
+ with sqlite3.connect(target) as connection:
+ connection.execute("create table alembic_version (version_num text primary key)")
+ connection.execute("insert into alembic_version values ('old')")
+ connection.execute("create table notes (body text)")
+ connection.execute("insert into notes values (?)", ("caf\u00e9",))
+ return target
+
+
+def _attempt(db_path: Path, *, phase: str = "ready", now: datetime = START, revision: str = "new") -> Path:
+ backup = backups.create_sqlite_migration_backup(db_path, to_revisions={revision}, now=now)
+ attempt = backups.begin_sqlite_backup_migration(db_path, backup)
+ if phase == "migrating":
+ return backup
+ with sqlite3.connect(db_path) as connection:
+ connection.execute("update alembic_version set version_num = ?", (revision,))
+ backups.complete_sqlite_backup_migration(db_path, attempt, now=now)
+ if phase == "migrated":
+ return backup
+ assert backups.begin_sqlite_backup_validation(db_path) == attempt
+ if phase == "validating":
+ return backup
+ backups.complete_sqlite_backup_validation(db_path, attempt, now=now + timedelta(seconds=10))
+ if phase == "validated":
+ return backup
+ assert backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=attempt, expected_revisions={revision}, now=now + timedelta(seconds=20)
+ )
+ return backup
+
+
+def _manifest(backup: Path) -> dict:
+ return json.loads((backup / "manifest.json").read_text(encoding="utf-8"))
+
+
+def _write_manifest(backup: Path, manifest: dict) -> None:
+ (backup / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
+
+
+ ("moment", "deleted"),
+ [
+ (EXPIRES - timedelta(microseconds=1), False),
+ (EXPIRES, True),
+ (EXPIRES + timedelta(days=10), True),
+ (EXPIRES.astimezone(timezone(timedelta(hours=8))), True),
+ ],
+)
+def test_expiry_boundary_can_remove_the_last_backup(tmp_path, moment, deleted):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+
+ result = backups.expire_sqlite_migration_backups(db_path, now=moment)
+
+ assert backup.exists() is not deleted
+ assert result["removed"] == ([backup.name] if deleted else [])
+ with sqlite3.connect(db_path) as connection:
+ assert connection.execute("select body from notes").fetchall() == [("caf\u00e9",)]
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(days=30))["removed"] == (
+ [backup.name] if not deleted else []
+ )
+
+
+def test_restart_does_not_reset_ready_time(tmp_path):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+ before = _manifest(backup)
+
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=before["expiry_lifecycle"]["attempt_id"],
+ expected_revisions={"new"}, now=READY + timedelta(days=2)
+ )
+ assert _manifest(backup) == before
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["removed"] == [backup.name]
+
+
+def test_count_limit_and_age_expiry_apply_independently(tmp_path):
+ db_path = _database(tmp_path)
+ copies = [
+ _attempt(db_path, now=START + timedelta(hours=index), revision=f"new-{index}")
+ for index in range(backups.SQLITE_BACKUP_RETENTION + 2)
+ ]
+ surviving = [backup for backup in copies if backup.exists()]
+ assert surviving == copies[-backups.SQLITE_BACKUP_RETENTION:]
+ assert backups.expire_sqlite_migration_backups(db_path, now=READY + timedelta(hours=5))["removed"] == []
+ result = backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(hours=5))
+ assert set(result["removed"]) == {backup.name for backup in surviving}
+ assert not any(backup.exists() for backup in copies)
+
+
[email protected]("phase", ["migrating", "migrated", "validating", "validated"])
+def test_unconfirmed_attempt_never_expires_by_age(tmp_path, phase):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path, phase=phase)
+
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(days=90)) == {
+ "status": "unconfirmed",
+ "removed": [],
+ }
+ assert backup.exists()
+
+
+def test_service_with_unrelated_target_cannot_confirm_attempt(tmp_path):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path, phase="validated")
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=_manifest(backup)["expiry_lifecycle"]["attempt_id"],
+ expected_revisions={"unrelated"}, now=READY
+ )
+ assert _manifest(backup)["expiry_lifecycle"]["phase"] == "validated"
+
+
+def test_readiness_cannot_adopt_a_later_attempt_with_the_same_revisions(tmp_path):
+ db_path = _database(tmp_path)
+ earlier = _attempt(db_path, phase="validated")
+ previous_attempt = _manifest(earlier)["expiry_lifecycle"]["attempt_id"]
+ later = _attempt(db_path, phase="validated", now=START + timedelta(days=1))
+
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=previous_attempt, expected_revisions={"new"}
+ )
+ assert _manifest(later)["expiry_lifecycle"]["phase"] == "validated"
+
+
+def test_new_unconfirmed_attempt_fences_older_expired_backups(tmp_path):
+ db_path = _database(tmp_path)
+ old = _attempt(db_path)
+ newer = backups.create_sqlite_migration_backup(
+ db_path, to_revisions={"future"}, now=READY + timedelta(days=4)
+ )
+ backups.begin_sqlite_backup_migration(db_path, newer)
+ # Simulate a migration that committed partial data, but did not advance its stamp.
+ with sqlite3.connect(db_path) as connection:
+ connection.execute("insert into notes values ('partial change')")
+
+ result = backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(days=1))
+
+ assert result["status"] == "unconfirmed"
+ assert old.exists() and newer.exists()
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=_manifest(old)["expiry_lifecycle"]["attempt_id"],
+ expected_revisions={"new"}, now=EXPIRES
+ )
+
+
+def test_failed_metadata_publication_cannot_leave_old_expiry_authority(monkeypatch, tmp_path):
+ db_path = _database(tmp_path)
+ old = _attempt(db_path)
+ newer = backups.create_sqlite_migration_backup(db_path, to_revisions={"future"})
+ real_write = backups._write_backup_metadata
+
+ def fail_manifest(path, payload):
+ if path == newer / "manifest.json":
+ raise OSError("disk full")
+ real_write(path, payload)
+
+ monkeypatch.setattr(backups, "_write_backup_metadata", fail_manifest)
+ with pytest.raises(OSError, match="disk full"):
+ backups.begin_sqlite_backup_migration(db_path, newer)
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["status"] == "unconfirmed"
+ assert old.exists()
+
+
+def test_partial_readiness_receipt_is_not_repaired_by_guessing(monkeypatch, tmp_path):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path, phase="validated")
+ real_write = backups._write_backup_metadata
+
+ def fail_record(path, payload):
+ if path == backups._lifecycle_path(db_path):
+ raise OSError("disk full")
+ real_write(path, payload)
+
+ monkeypatch.setattr(backups, "_write_backup_metadata", fail_record)
+ with pytest.raises(OSError, match="disk full"):
+ backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=_manifest(backup)["expiry_lifecycle"]["attempt_id"],
+ expected_revisions={"new"}, now=READY
+ )
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["status"] == "unconfirmed"
+ assert backup.exists()
+
+
+ ("field", "value"),
+ [
+ ("schema_version", 2),
+ ("schema_version", True),
+ ("attempt_id", "invalid"),
+ ("phase", "restored"),
+ ("phase", []),
+ ("phase", {}),
+ ("database", "/unrelated/vibe.sqlite"),
+ ("database_identity", [False, True]),
+ ("backup_name", "../another"),
+ ("migration_completed_at", None),
+ ("migration_completed_at", "not-a-date"),
+ ("validated_at", "2026-09-28T11:59:00+00:00"),
+ ("ready_at", "2026-09-28T12:00:20"),
+ ("ready_at", "2099-01-01T00:00:00Z"),
+ ("ready_at", "2026-09-28T12:00:00Z"),
+ ],
+)
+def test_malformed_expiry_evidence_is_not_deletion_authority(tmp_path, field, value):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+ manifest = _manifest(backup)
+ manifest["expiry_lifecycle"][field] = value
+ _write_manifest(backup, manifest)
+
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["removed"] == []
+ assert backup.exists()
+
+
[email protected]("payload", [b"\xff", b'{"phase": []}', b'{"phase": {}}'])
+def test_unreadable_or_malformed_sidecar_does_not_block_database_use(tmp_path, payload):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+ backups._lifecycle_path(db_path).write_bytes(payload)
+
+ assert backups.begin_sqlite_backup_validation(db_path) is None
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=_manifest(backup)["expiry_lifecycle"]["attempt_id"], expected_revisions={"new"}
+ )
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["status"] == "unconfirmed"
+ assert backup.exists()
+
+
+def test_unknown_legacy_json_and_manual_files_are_not_age_expired(tmp_path):
+ db_path = _database(tmp_path)
+ ready = _attempt(db_path)
+ legacy = backups.create_sqlite_migration_backup(db_path, now=START - timedelta(days=300))
+ root = ready.parent
+ manual = root / "manual-keep.sqlite"
+ manual.write_text("user data", encoding="utf-8")
+ json_backup = root / "sqlite-state-migration-20260101T000000Z"
+ json_backup.mkdir()
+ (json_backup / "manifest.json").write_text(
+ json.dumps({"created_at": "2026-01-01T00:00:00Z", "files": {}}), encoding="utf-8"
+ )
+
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["removed"] == [ready.name]
+ assert legacy.exists()
+ assert manual.read_text(encoding="utf-8") == "user data"
+ assert json_backup.exists()
+
+
[email protected]("extra_name", [backups.REPLACED_DATABASE_NAME, "operator-notes.txt"])
+def test_user_added_or_displaced_content_is_not_age_expired(tmp_path, extra_name):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+ extra = backup / extra_name
+ extra.write_text("not temporary", encoding="utf-8")
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["removed"] == []
+ assert extra.read_text(encoding="utf-8") == "not temporary"
+
+
+def test_symlink_backup_root_is_not_traversed(tmp_path):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+ actual_root = tmp_path / "external"
+ backup.parent.rename(actual_root)
+ backup.parent.symlink_to(actual_root, target_is_directory=True)
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["removed"] == []
+ assert (actual_root / backup.name).exists()
+
+
+def test_restore_revokes_all_prior_expiry_and_preserves_displaced_writes(tmp_path):
+ db_path = _database(tmp_path)
+ old = _attempt(db_path)
+ latest = _attempt(db_path, now=START + timedelta(days=1))
+ with sqlite3.connect(db_path) as connection:
+ connection.execute("insert into notes values ('after backup')")
+
+ displaced = backups.restore_sqlite_backup(old, db_path)
+
+ assert displaced is not None
+ with sqlite3.connect(displaced) as connection:
+ assert connection.execute("select body from notes where body = 'after backup'").fetchone()
+ assert _manifest(old)["expiry_lifecycle"]["phase"] == "restored"
+ assert _manifest(latest)["expiry_lifecycle"]["phase"] == "restored"
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(days=10))["removed"] == []
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=_manifest(old)["expiry_lifecycle"]["attempt_id"], expected_revisions={"old"}
+ )
+
+
+def test_failed_restore_does_not_keep_an_old_expiry_clock(monkeypatch, tmp_path):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+
+ def fail_swap(*args, **kwargs):
+ raise OSError("restore failed")
+
+ monkeypatch.setattr(backups, "_swap_live_database", fail_swap)
+ with pytest.raises(OSError, match="restore failed"):
+ backups.restore_sqlite_backup(backup, db_path)
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["status"] == "unconfirmed"
+ assert backup.exists()
+
+
+def test_expiry_defers_when_migration_lock_is_owned_by_another_thread(tmp_path):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+ acquired = threading.Event()
+ release = threading.Event()
+
+ def hold_lock():
+ with MigrationFileLock(migration_lock_path_for(db_path), timeout_seconds=None):
+ acquired.set()
+ assert release.wait(10)
+
+ worker = threading.Thread(target=hold_lock)
+ worker.start()
+ try:
+ assert acquired.wait(10)
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["status"] == "busy"
+ assert backup.exists()
+ finally:
+ release.set()
+ worker.join(10)
+ assert not worker.is_alive()
+
+
+def test_replaced_database_cannot_inherit_previous_expiry_receipts(tmp_path):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+ replacement = db_path.with_name("replacement.sqlite")
+ with sqlite3.connect(db_path) as source, sqlite3.connect(replacement) as target:
+ source.backup(target)
+ replacement.replace(db_path)
+
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES)["status"] == "unconfirmed"
+ assert backup.exists()
+
+
+def test_cancelled_expiry_does_not_delete(tmp_path):
+ db_path = _database(tmp_path)
+ backup = _attempt(db_path)
+ cancelled = threading.Event()
+ cancelled.set()
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES, cancel_event=cancelled)["status"] == "cancelled"
+ assert backup.exists()
+
+
+def test_real_migration_import_and_service_readiness_enable_expiry(monkeypatch, tmp_path):
+ from config import paths
+ from vibe import runtime
+
+ state_dir = tmp_path / "state"
+ db_path = state_dir / "vibe.sqlite"
+ state_dir.mkdir()
+ run_migrations(db_path, revision="20260627_0025")
+ ensure_sqlite_state(db_path=db_path, state_dir=state_dir, primary_platform="avibe")
+ record = json.loads(backups._lifecycle_path(db_path).read_text(encoding="utf-8"))
+ backup = state_dir / "backups" / record["backup_name"]
+ assert record["phase"] == "validated"
+ assert backups.expire_sqlite_migration_backups(db_path)["status"] == "unconfirmed"
+ monkeypatch.setattr(paths, "get_sqlite_state_path", lambda: db_path)
+
+ # The real runtime owner, not a synthetic "healthy" flag, confirms readiness.
+ runtime.acquire_service_instance_lock()
+ try:
+ runtime.mark_service_instance_started()
+ ready = datetime.fromisoformat(_manifest(backup)["expiry_lifecycle"]["ready_at"])
+ runtime.mark_service_instance_started()
+ assert datetime.fromisoformat(_manifest(backup)["expiry_lifecycle"]["ready_at"]) == ready
+ assert backups.expire_sqlite_migration_backups(db_path, now=ready + timedelta(hours=72))["removed"] == [
+ backup.name
+ ]
+ finally:
+ runtime.release_service_instance_lock()
+ with sqlite3.connect(db_path) as connection:
+ assert connection.execute("PRAGMA quick_check").fetchone() == ("ok",)
+
+
+def test_busy_readiness_retries_on_maintenance_without_renewing_clock(monkeypatch, tmp_path):
+ from config import paths
+ from core.controller import Controller
+ from vibe import runtime
+
+ state_dir = tmp_path / "state"
+ state_dir.mkdir()
+ db_path = state_dir / "vibe.sqlite"
+ run_migrations(db_path, revision="20260627_0025")
+ report = ensure_sqlite_state(db_path=db_path, state_dir=state_dir, primary_platform="avibe")
+ assert report.backup_attempt_id
+ record = json.loads(backups._lifecycle_path(db_path).read_text(encoding="utf-8"))
+ backup = state_dir / "backups" / record["backup_name"]
+ monkeypatch.setattr(paths, "get_sqlite_state_path", lambda: db_path)
+ acquired = threading.Event()
+ release = threading.Event()
+
+ def hold_lock():
+ with MigrationFileLock(migration_lock_path_for(db_path), timeout_seconds=None):
+ acquired.set()
+ assert release.wait(10)
+
+ worker = threading.Thread(target=hold_lock)
+ worker.start()
+ runtime.acquire_service_instance_lock()
+ try:
+ assert acquired.wait(10)
+ runtime.mark_service_instance_started()
+ pending = runtime._PENDING_SQLITE_BACKUP_READINESS
+ assert pending is not None
+ original_ready_at = pending[-1]
+ assert _manifest(backup)["expiry_lifecycle"]["phase"] == "validated"
+ release.set()
+ worker.join(10)
+ real_expiry = backups.expire_sqlite_migration_backups
+ observed = []
+
+ def expiry_after_deadline(path, **kwargs):
+ observed.append(_manifest(backup)["expiry_lifecycle"]["ready_at"])
+ return real_expiry(path, now=original_ready_at + timedelta(hours=72), **kwargs)
+
+ monkeypatch.setattr(backups, "expire_sqlite_migration_backups", expiry_after_deadline)
+ controller = Controller.__new__(Controller)
+ assert controller._run_sqlite_backup_expiry_pass()["removed"] == [backup.name]
+ assert observed == [original_ready_at.isoformat()]
+ assert runtime._PENDING_SQLITE_BACKUP_READINESS is None
+ finally:
+ release.set()
+ worker.join(10)
+ runtime.release_service_instance_lock()
+ assert not worker.is_alive()
+
+
+def test_service_cannot_adopt_another_process_validation(monkeypatch, tmp_path):
+ from config import paths
+ from storage.importer import reset_ensured_sqlite_state
+ from vibe import runtime
+
+ state_dir = tmp_path / "state"
+ state_dir.mkdir()
+ db_path = state_dir / "vibe.sqlite"
+ run_migrations(db_path, revision="20260627_0025")
+ ensure_sqlite_state(db_path=db_path, state_dir=state_dir, primary_platform="avibe")
+ reset_ensured_sqlite_state()
+ monkeypatch.setattr(paths, "get_sqlite_state_path", lambda: db_path)
+ runtime.acquire_service_instance_lock()
+ try:
+ runtime.mark_service_instance_started()
+ record = json.loads(backups._lifecycle_path(db_path).read_text(encoding="utf-8"))
+ assert record["phase"] == "validated"
+ assert runtime._PENDING_SQLITE_BACKUP_READINESS is None
+ finally:
+ runtime.release_service_instance_lock()
+
+
[email protected]("phase", ["migrated", "validating", "validated"])
[email protected]("receipt", ["manifest", "sidecar"])
+def test_optional_receipt_failure_retains_backup_without_failing_startup(monkeypatch, tmp_path, phase, receipt):
+ state_dir = tmp_path / "state"
+ state_dir.mkdir()
+ db_path = state_dir / "vibe.sqlite"
+ run_migrations(db_path, revision="20260627_0025")
+ real_write = backups._write_backup_metadata
+ failures = []
+
+ def fail_receipt(path, payload):
+ lifecycle = payload.get("expiry_lifecycle", payload)
+ is_manifest = path.name == "manifest.json"
+ if lifecycle.get("phase") == phase and is_manifest == (receipt == "manifest"):
+ failures.append(path)
+ raise OSError("disk full while publishing optional receipt")
+ real_write(path, payload)
+
+ monkeypatch.setattr(backups, "_write_backup_metadata", fail_receipt)
+ report = ensure_sqlite_state(db_path=db_path, state_dir=state_dir, primary_platform="avibe")
+
+ assert report.db_path == db_path
+ assert failures
+ record = json.loads(backups._lifecycle_path(db_path).read_text(encoding="utf-8"))
+ backup = state_dir / "backups" / record["backup_name"]
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=record["attempt_id"], expected_revisions=record["to_revisions"]
+ )
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(days=90))["removed"] == []
+ assert backup.exists()
+ with sqlite3.connect(db_path) as connection:
+ assert connection.execute("PRAGMA quick_check").fetchone() == ("ok",)
+
+
+def test_initial_fence_failure_aborts_before_sql_changes(monkeypatch, tmp_path):
+ from storage import migrations
+
+ state_dir = tmp_path / "state"
+ state_dir.mkdir()
+ db_path = state_dir / "vibe.sqlite"
+ run_migrations(db_path, revision="20260627_0025")
+ real_write = backups._write_backup_metadata
+
+ def fail_fence(path, payload):
+ if path == backups._lifecycle_path(db_path):
+ raise OSError("cannot invalidate prior expiry authority")
+ real_write(path, payload)
+
+ def unexpected_upgrade(*args, **kwargs):
+ pytest.fail("The database must not change before the blocking record is durable")
+
+ monkeypatch.setattr(backups, "_write_backup_metadata", fail_fence)
+ monkeypatch.setattr(migrations.command, "upgrade", unexpected_upgrade)
+ with pytest.raises(OSError, match="cannot invalidate"):
+ run_migrations(db_path)
+ with sqlite3.connect(db_path) as connection:
+ assert backups._stamped_revisions(connection) == ("20260627_0025",)
+
+
+def test_relocated_backup_directory_keeps_upgrades_working_without_age_expiry(tmp_path):
+ state_dir = tmp_path / "state"
+ state_dir.mkdir()
+ db_path = state_dir / "vibe.sqlite"
+ run_migrations(db_path, revision="20260627_0025")
+ external = tmp_path / "external-backups"
+ external.mkdir()
+ (state_dir / "backups").symlink_to(external, target_is_directory=True)
+
+ report = ensure_sqlite_state(db_path=db_path, state_dir=state_dir, primary_platform="avibe")
+
+ assert report.db_path == db_path
+ copies = [entry for entry in external.iterdir() if entry.name.startswith("avibe-sqlite-migration-")]
+ assert len(copies) == 1
+ assert "expiry_lifecycle" not in _manifest(copies[0])
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(days=90))["removed"] == []
+ with sqlite3.connect(db_path) as connection:
+ assert connection.execute("PRAGMA quick_check").fetchone() == ("ok",)
+
+
+def test_import_failure_keeps_migrated_backup_unconfirmed(monkeypatch, tmp_path):
+ from storage import importer
+
+ state_dir = tmp_path / "state"
+ state_dir.mkdir()
+ db_path = state_dir / "vibe.sqlite"
+ run_migrations(db_path, revision="20260627_0025")
+
+ def fail_import(*args, **kwargs):
+ raise RuntimeError("data import failed")
+
+ monkeypatch.setattr(importer, "_parse_json_state", fail_import)
+ with pytest.raises(RuntimeError, match="data import failed"):
+ ensure_sqlite_state(db_path=db_path, state_dir=state_dir, primary_platform="avibe")
+ record = json.loads(backups._lifecycle_path(db_path).read_text(encoding="utf-8"))
+ assert record["phase"] == "validating"
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(days=30))["removed"] == []
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=record["attempt_id"], expected_revisions=record["to_revisions"]
+ )
+
+
+def test_real_failed_migration_cannot_gain_readiness_from_unchanged_revision(monkeypatch, tmp_path):
+ from storage import migrations
+
+ state_dir = tmp_path / "state"
+ state_dir.mkdir()
+ db_path = state_dir / "vibe.sqlite"
+ run_migrations(db_path, revision="20260627_0025")
+
+ def fail_upgrade(*args, **kwargs):
+ with sqlite3.connect(db_path) as connection:
+ connection.execute("create table partial_write (value text)")
+ raise RuntimeError("partial migration")
+
+ monkeypatch.setattr(migrations.command, "upgrade", fail_upgrade)
+ with pytest.raises(RuntimeError, match="partial migration"):
+ run_migrations(db_path)
+ record = json.loads(backups._lifecycle_path(db_path).read_text(encoding="utf-8"))
+ assert record["phase"] == "migrating"
+ assert not backups.confirm_sqlite_backup_readiness(
+ db_path, attempt_id=record["attempt_id"], expected_revisions=record["to_revisions"]
+ )
+ assert backups.expire_sqlite_migration_backups(db_path, now=EXPIRES + timedelta(days=30))["removed"] == []
diff --git a/vibe/runtime.py b/vibe/runtime.py
index 0ec856b80..6d52140e0 100644
--- a/vibe/runtime.py
+++ b/vibe/runtime.py
@@ -16,6 +16,7 @@ import urllib.error
import urllib.request
import uuid
from dataclasses import dataclass
+from datetime import datetime, timezone
from pathlib import Path
from typing import Callable, NamedTuple
@@ -162,6 +163,7 @@ ROOT_DIR = get_project_root() # For backward compatibility
MAIN_PATH = get_service_main_path()
_SERVICE_LOCK = threading.Lock()
_SERVICE_INSTANCE_LOCK_HANDLE = None
+_PENDING_SQLITE_BACKUP_READINESS: tuple[Path, str, tuple[str, ...], datetime] | None = None
_SERVICE_START_PROCESSES: dict[int, subprocess.Popen] = {}
# /ready can spend up to two seconds in the Controller health probe.
UI_ADOPTION_PROBE_TIMEOUT_SECONDS = 3.0
@@ -400,6 +402,45 @@ def mark_service_instance_started() -> None:
if lock_file is None:
return
_write_service_instance_lock_record(lock_file, SERVICE_PHASE_RUNNING)
+ global _PENDING_SQLITE_BACKUP_READINESS
+ try:
+ from alembic.script import ScriptDirectory
+
+ from storage.importer import validated_sqlite_backup_attempt
+ from storage.migrations import alembic_config
+
+ target = paths.get_sqlite_state_path()
+ attempt = validated_sqlite_backup_attempt(target)
+ if attempt is not None and _PENDING_SQLITE_BACKUP_READINESS is None:
+ _PENDING_SQLITE_BACKUP_READINESS = (
+ target,
+ attempt,
+ tuple(ScriptDirectory.from_config(alembic_config(target)).get_heads()),
+ datetime.now(timezone.utc),
+ )
+ confirm_pending_sqlite_backup_readiness()
+ except Exception:
+ # Missing expiry evidence retains the bounded rollback window. Optional
+ # maintenance must not turn an otherwise ready service into a failed one.
+ logger.warning("Could not confirm SQLite backup expiry readiness; retaining backups", exc_info=True)
+
+
+def confirm_pending_sqlite_backup_readiness() -> None:
+ """Retry only this service's observed readiness, preserving its original clock."""
+ global _PENDING_SQLITE_BACKUP_READINESS
+ pending = _PENDING_SQLITE_BACKUP_READINESS
+ if pending is None or not current_process_owns_service_instance() or not service_instance_started(os.getpid()):
+ return
+ from storage.backups import confirm_sqlite_backup_readiness
+
+ target, attempt, revisions, ready_at = pending
+ confirm_sqlite_backup_readiness(
+ target, attempt_id=attempt, expected_revisions=revisions, now=ready_at
+ )
+ # A busy lock raises and keeps the pending event. A false result means this
+ # exact attempt is no longer eligible; never adopt another process's attempt.
+ if _PENDING_SQLITE_BACKUP_READINESS is pending:
+ _PENDING_SQLITE_BACKUP_READINESS = None
def read_service_instance_lock_record() -> dict | None:
@@ -455,7 +496,8 @@ def service_instance_still_starting(pid: int) -> bool:
def release_service_instance_lock() -> None:
- global _SERVICE_INSTANCE_LOCK_HANDLE
+ global _SERVICE_INSTANCE_LOCK_HANDLE, _PENDING_SQLITE_BACKUP_READINESS
+ _PENDING_SQLITE_BACKUP_READINESS = None
lock_file = _SERVICE_INSTANCE_LOCK_HANDLE
if lock_file is None:
return