Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.
Rust ai-agentsai-gatewayclaude-codecodexcoding-agentcost-optimizationdeepseekllm-gatewayllm-routerlocal-firstmodel-routingrusttask-routingtauri
#[cfg(test)]
mod digest_tests;
pub(crate) mod source_snapshot;
use std::fs;
use std::path::{Path, PathBuf};
use hiroute_domain::CanonicalDigest;
use rusqlite::types::ValueRef;
use rusqlite::{Connection, OpenFlags, OptionalExtension, params};
use serde::{Deserialize, Serialize};
use sha2::{Digest as _, Sha256};
use crate::{DaemonStorageAuthority, LocalStorageError};
const BACKUP_MANIFEST_SCHEMA: &str = "hiroute.sqlite-backup-manifest/v2";
const BACKUP_SET_MANIFEST_SCHEMA: &str = "hiroute.sqlite-backup-set/v2";
const BACKUP_SET_MANIFEST_NAME: &str = "backup-set.manifest.json";
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BackupManifest {
pub schema: String,
pub database_kind: String,
pub store_uuid: String,
pub schema_version: u32,
pub source_digest: CanonicalDigest,
pub source_state_digest: CanonicalDigest,
pub key_id: Option<String>,
pub journal_watermark: String,
pub attempt_id: String,
}
#[derive(Clone, Debug)]
pub struct SqliteBackup {
path: PathBuf,
manifest: BackupManifest,
}
impl SqliteBackup {
pub fn create(
_authority: &DaemonStorageAuthority,
connection: &Connection,
path: impl AsRef<Path>,
) -> Result<Self, LocalStorageError> {
Self::create_bound(connection, path, None)
}
pub(crate) fn create_bound(
connection: &Connection,
path: impl AsRef<Path>,
key_id: Option<&CanonicalDigest>,
) -> Result<Self, LocalStorageError> {
let watermark = standalone_watermark(connection)?;
Self::create_at_watermark(connection, path.as_ref(), &watermark, key_id)
}
fn create_at_watermark(
connection: &Connection,
path: &Path,
journal_watermark: &str,
key_id: Option<&CanonicalDigest>,
) -> Result<Self, LocalStorageError> {
let parent = path.parent().ok_or(LocalStorageError::InvalidData)?;
crate::prepare_owner_directory(parent)?;
let attempt_id = random_attempt_id()?;
let file_name = path
.file_name()
.and_then(|name| name.to_str())
.ok_or(LocalStorageError::InvalidData)?;
// On Unix, `VACUUM INTO` creates the destination before this process can tighten its mode. A crash
// in that small window leaves a current-uid regular file in an already owner-only
// directory. Reconcile those exact attempt names before allocating the next attempt so a
// restart never treats a recoverable 0644 SQLite stage as a permanent permission fault.
cleanup_orphan_stages(parent, file_name)?;
let stage = parent.join(format!(".{file_name}.{attempt_id}.stage.db"));
let stage_manifest = parent.join(format!(".{file_name}.{attempt_id}.stage.manifest.json"));
if stage.exists() || stage_manifest.exists() {
return Err(LocalStorageError::InvalidData);
}
connection.execute_batch("PRAGMA wal_checkpoint(FULL)")?;
let source_state_digest = database_state_digest(connection)?;
// SQLite explicitly accepts an existing empty INTO file. Allocate it
// privately first so an elevated default token owner never reaches the
// backup, including during the crash window before VACUUM completes.
#[cfg(windows)]
hiroute_platform::durable::write_private_new(&stage, b"")?;
connection.execute(
"VACUUM main INTO ?1",
params![stage.to_string_lossy().as_ref()],
)?;
set_owner_file_permissions(&stage)?;
hiroute_platform::durable::sync_file(&stage)?;
validate_database(&stage)?;
let identity = database_identity(connection)?;
if identity
.key_id
.as_deref()
.is_some_and(|stored| key_id.is_some_and(|requested| stored != requested.as_str()))
{
return Err(LocalStorageError::Locked);
}
let manifest = BackupManifest {
schema: BACKUP_MANIFEST_SCHEMA.to_owned(),
database_kind: identity.kind,
store_uuid: identity.store_uuid,
schema_version: identity.schema_version,
source_digest: file_digest(&stage)?,
source_state_digest,
key_id: identity.key_id.or_else(|| key_id.map(ToString::to_string)),
journal_watermark: journal_watermark.to_owned(),
attempt_id: attempt_id.clone(),
};
write_manifest(&stage_manifest, &manifest)?;
hiroute_platform::durable::directory_barrier(parent)?;
let final_manifest = manifest_path(path);
if path.exists() || final_manifest.exists() {
if path.exists()
&& final_manifest.exists()
&& Self::open(path).is_ok_and(|existing| {
manifest_exact_except_attempt(&existing.manifest, &manifest)
})
{
fs::remove_file(&stage)?;
fs::remove_file(&stage_manifest)?;
cleanup_orphan_stages(parent, file_name)?;
return Self::open(path);
}
// Preserve stale or half-published finals for forensic recovery; they never block a
// fresh exact attempt and are never silently reused for a different source snapshot.
if path.exists() {
hiroute_platform::durable::rename(
path,
parent.join(format!(".{file_name}.{attempt_id}.stale.db")),
)?;
}
if final_manifest.exists() {
hiroute_platform::durable::rename(
&final_manifest,
parent.join(format!(".{file_name}.{attempt_id}.stale.manifest.json")),
)?;
}
hiroute_platform::durable::directory_barrier(parent)?;
}
hiroute_platform::durable::rename(&stage, path)?;
hiroute_platform::durable::directory_barrier(parent)?;
hiroute_platform::durable::rename(&stage_manifest, &final_manifest)?;
hiroute_platform::durable::directory_barrier(parent)?;
let backup = Self::open(path)?;
cleanup_orphan_stages(parent, file_name)?;
Ok(backup)
}
pub fn open(path: impl AsRef<Path>) -> Result<Self, LocalStorageError> {
let path = path.as_ref();
validate_owner_file(path)?;
validate_database(path)?;
let manifest_path = manifest_path(path);
validate_owner_file(&manifest_path)?;
let manifest: BackupManifest = serde_json::from_slice(&fs::read(&manifest_path)?)
.map_err(|_| LocalStorageError::InvalidData)?;
if manifest.schema != BACKUP_MANIFEST_SCHEMA || manifest.source_digest != file_digest(path)?
{
return Err(LocalStorageError::InvalidData);
}
let connection = Connection::open_with_flags(
path,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
let identity = database_identity(&connection)?;
if manifest.database_kind != identity.kind
|| manifest.store_uuid != identity.store_uuid
|| manifest.schema_version != identity.schema_version
|| identity
.key_id
.as_ref()
.is_some_and(|key_id| manifest.key_id.as_ref() != Some(key_id))
|| manifest
.key_id
.as_deref()
.is_some_and(|key_id| CanonicalDigest::parse(key_id).is_err())
|| manifest.source_state_digest != database_state_digest(&connection)?
{
return Err(LocalStorageError::InvalidData);
}
Ok(Self {
path: path.to_path_buf(),
manifest,
})
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn manifest(&self) -> &BackupManifest {
&self.manifest
}
/// Restore is intentionally create-new. The exact DB and its manifest are verified before and
/// after publication; replacing a live database remains a separate reversible owner action.
pub(crate) fn restore_to(
&self,
destination: impl AsRef<Path>,
) -> Result<(), LocalStorageError> {
Self::open(&self.path)?;
let destination = destination.as_ref();
if destination.exists() {
return Err(LocalStorageError::InvalidData);
}
let parent = destination.parent().ok_or(LocalStorageError::InvalidData)?;
crate::prepare_owner_directory(parent)?;
let attempt = random_attempt_id()?;
let temporary = parent.join(format!(".restore-{attempt}.stage.db"));
#[cfg(windows)]
{
use hiroute_platform::windows::security;
let mut source = security::open_owner_only(&self.path, false)?;
let mut target = security::create_owner_only_file(&temporary)?;
std::io::copy(&mut source, &mut target)?;
target.sync_all()?;
security::verify_owner_only_handle(&target, false)?;
}
#[cfg(not(windows))]
fs::copy(&self.path, &temporary)?;
set_owner_file_permissions(&temporary)?;
hiroute_platform::durable::sync_file(&temporary)?;
validate_database(&temporary)?;
if file_digest(&temporary)? != self.manifest.source_digest {
return Err(LocalStorageError::InvalidData);
}
hiroute_platform::durable::rename(&temporary, destination)?;
hiroute_platform::durable::directory_barrier(parent)?;
validate_database(destination)?;
if file_digest(destination)? != self.manifest.source_digest {
return Err(LocalStorageError::InvalidData);
}
Ok(())
}
}
#[derive(Clone, Debug)]
pub struct BackupSet {
pub control: SqliteBackup,
pub runtime: SqliteBackup,
pub secrets: SqliteBackup,
root: PathBuf,
set_id: String,
attempt_id: String,
phase: BackupSetPhase,
target_schema_version: u32,
watermark: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BackupSetPhase {
SourcePublished,
ControlMigrated,
RuntimeMigrated,
SecretsMigrated,
Completed,
Restoring,
Restored,
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
struct BackupSetManifest {
schema: String,
set_id: String,
attempt_id: String,
phase: BackupSetPhase,
target_schema_version: u32,
watermark: String,
control: BackupManifest,
runtime: BackupManifest,
secrets: BackupManifest,
}
/// Proof that the daemon-owned writer actor is paused for the full three-store snapshot.
///
/// The field is intentionally private and no constructor is exposed in this PROCESS. The formal
/// daemon composition owner will mint it only while holding the same lock used by transaction
/// admission; raw callers cannot claim a consistent set from three unrelated connections.
pub struct SingleWriterBackupBarrier {
_private: (),
}
/// Crate-sealed proof used only while daemon startup still has listeners and transaction
/// admission closed. No production caller can mint or retain the raw barrier.
pub(crate) fn daemon_startup_barrier() -> SingleWriterBackupBarrier {
SingleWriterBackupBarrier { _private: () }
}
#[cfg(test)]
pub(crate) fn test_writer_barrier() -> SingleWriterBackupBarrier {
daemon_startup_barrier()
}
impl BackupSet {
/// Saves the complete stopped storage authority beside this verified three-store backup.
/// The caller must retain the startup barrier and directory ownership until publication ends.
pub fn publish_stopped_source(
&self,
barrier: &SingleWriterBackupBarrier,
storage_root: &Path,
options: &crate::StorageStartupOptions,
) -> Result<(), LocalStorageError> {
if self
.root
.canonicalize()?
.starts_with(storage_root.canonicalize()?)
{
return Err(LocalStorageError::InvalidData);
}
for (name, backup) in [
("control.db", &self.control),
("runtime.db", &self.runtime),
("secrets.db", &self.secrets),
] {
let connection = Connection::open_with_flags(
storage_root.join("live").join(name),
OpenFlags::SQLITE_OPEN_READ_ONLY,
)?;
validate_restored_source_identity(&connection, backup.manifest())?;
if database_state_digest(&connection)? != backup.manifest().source_state_digest {
return Err(LocalStorageError::InvalidData);
}
}
self.publish_coordinated_source(barrier, storage_root, options)
}
/// The SQL coordinator admits only its verified source/target phases. Those
/// phases have never opened business or Observation writers, so unchanged
/// non-core state can be paired with the original three BackupSet sources
/// even when an older binary committed a SQL step before it stopped.
pub(crate) fn publish_coordinated_source(
&self,
_barrier: &SingleWriterBackupBarrier,
storage_root: &Path,
options: &crate::StorageStartupOptions,
) -> Result<(), LocalStorageError> {
if self
.root
.canonicalize()?
.starts_with(storage_root.canonicalize()?)
{
return Err(LocalStorageError::InvalidData);
}
let metadata = serde_json::json!({"batch_id":self.set_id(),"target_schema_version":self.target_schema_version(),"source_stores":{"control":self.control.manifest(),"runtime":self.runtime.manifest(),"secrets":self.secrets.manifest()},"source_group":"original_core_batch_and_stopped_non_core_state"});
source_snapshot::publish(storage_root, &self.root, &metadata, options)
}
pub(crate) fn directory(&self) -> &Path {
&self.root
}
/// Creates a local three-store snapshot under the hirouted single-writer barrier. A durable
/// writer claim proves that the barrier is not held and causes a fail-closed refusal.
pub fn create(
barrier: &SingleWriterBackupBarrier,
backup_root: impl AsRef<Path>,
control: &Connection,
runtime: &Connection,
secrets: &Connection,
) -> Result<Self, LocalStorageError> {
Self::create_inner(
barrier,
backup_root.as_ref(),
control,
runtime,
secrets,
None,
crate::migrations::LATEST_SCHEMA_VERSION,
false,
)
}
pub(crate) fn create_migration(
barrier: &SingleWriterBackupBarrier,
backup_root: &Path,
control: &Connection,
runtime: &Connection,
secrets: &Connection,
secret_key_id: &CanonicalDigest,
target_schema_version: u32,
) -> Result<Self, LocalStorageError> {
Self::create_inner(
barrier,
backup_root,
control,
runtime,
secrets,
Some(secret_key_id),
target_schema_version,
true,
)
}
#[allow(clippy::too_many_arguments)]
fn create_inner(
_barrier: &SingleWriterBackupBarrier,
backup_root: &Path,
control: &Connection,
runtime: &Connection,
secrets: &Connection,
secret_key_id: Option<&CanonicalDigest>,
target_schema_version: u32,
startup_migration: bool,
) -> Result<Self, LocalStorageError> {
crate::prepare_owner_directory(backup_root)?;
if reconcile_set_manifest_publication(backup_root)? {
let existing = Self::open_inner(backup_root)?;
if existing.target_schema_version != target_schema_version
|| !existing.sources_match_connections(control, runtime, secrets, secret_key_id)?
{
return Err(LocalStorageError::InvalidData);
}
return Ok(existing);
}
let writer_table: bool = control.query_row(
"SELECT EXISTS(
SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'writer_claim'
)",
[],
|row| row.get(0),
)?;
let writer_claim = writer_table
&& control.query_row("SELECT EXISTS(SELECT 1 FROM writer_claim)", [], |row| {
row.get::<_, bool>(0)
})?;
if writer_claim && !startup_migration {
return Err(LocalStorageError::InvalidData);
}
control.execute_batch("PRAGMA wal_checkpoint(FULL)")?;
runtime.execute_batch("PRAGMA wal_checkpoint(FULL)")?;
secrets.execute_batch("PRAGMA wal_checkpoint(FULL)")?;
let watermark = control_watermark(control)?;
let control = SqliteBackup::create_at_watermark(
control,
&backup_root.join("control.db"),
&watermark,
None,
)?;
let runtime = SqliteBackup::create_at_watermark(
runtime,
&backup_root.join("runtime.db"),
&watermark,
None,
)?;
let secrets = SqliteBackup::create_at_watermark(
secrets,
&backup_root.join("secrets.db"),
&watermark,
secret_key_id,
)?;
if control.manifest.journal_watermark != runtime.manifest.journal_watermark
|| control.manifest.journal_watermark != secrets.manifest.journal_watermark
{
return Err(LocalStorageError::InvalidData);
}
let manifest = BackupSetManifest {
schema: BACKUP_SET_MANIFEST_SCHEMA.to_owned(),
set_id: format!("set_{}", random_attempt_id()?),
attempt_id: random_attempt_id()?,
phase: BackupSetPhase::SourcePublished,
target_schema_version,
watermark,
control: control.manifest.clone(),
runtime: runtime.manifest.clone(),
secrets: secrets.manifest.clone(),
};
publish_set_manifest(backup_root, &manifest, false)?;
Self::from_manifest(backup_root, manifest)
}
/// Reopens a durable three-store source set after process loss and validates every exact
/// per-store backup before returning the handle.
pub fn open(
_authority: &DaemonStorageAuthority,
backup_root: impl AsRef<Path>,
) -> Result<Self, LocalStorageError> {
Self::open_inner(backup_root.as_ref())
}
pub(crate) fn open_inner(backup_root: &Path) -> Result<Self, LocalStorageError> {
crate::prepare_owner_directory(backup_root)?;
if !reconcile_set_manifest_publication(backup_root)? {
return Err(LocalStorageError::InvalidData);
}
let path = set_manifest_path(backup_root);
validate_owner_file(&path)?;
let manifest: BackupSetManifest =
serde_json::from_slice(&fs::read(path)?).map_err(|_| LocalStorageError::InvalidData)?;
Self::from_manifest(backup_root, manifest)
}
pub(crate) fn durable_manifest_present(backup_root: &Path) -> Result<bool, LocalStorageError> {
if !backup_root.exists() {
return Ok(false);
}
crate::prepare_owner_directory(backup_root)?;
reconcile_set_manifest_publication(backup_root)
}
fn from_manifest(
backup_root: &Path,
manifest: BackupSetManifest,
) -> Result<Self, LocalStorageError> {
validate_set_manifest_shape(&manifest)?;
let (control, runtime, secrets) =
validate_manifest_against_backups(backup_root, &manifest)?;
Ok(Self {
control,
runtime,
secrets,
root: backup_root.to_path_buf(),
set_id: manifest.set_id,
attempt_id: manifest.attempt_id,
phase: manifest.phase,
target_schema_version: manifest.target_schema_version,
watermark: manifest.watermark,
})
}
pub fn watermark(&self) -> &str {
&self.watermark
}
pub fn set_id(&self) -> &str {
&self.set_id
}
pub const fn phase(&self) -> BackupSetPhase {
self.phase
}
pub const fn target_schema_version(&self) -> u32 {
self.target_schema_version
}
pub(crate) fn persist_phase(
&mut self,
_barrier: &SingleWriterBackupBarrier,
phase: BackupSetPhase,
) -> Result<(), LocalStorageError> {
if phase == self.phase {
return Ok(());
}
if !valid_phase_transition(self.phase, phase) {
return Err(LocalStorageError::InvalidData);
}
let mut manifest = self.set_manifest();
manifest.phase = phase;
publish_set_manifest(&self.root, &manifest, true)?;
self.phase = phase;
Ok(())
}
/// Restores the already-validated set into one new directory. All three databases and their
/// manifests are prepared beneath a unique sibling directory, fsynced, re-opened, and only
/// then published with one directory rename. A crash cannot expose a partial final set.
pub fn restore_to_new_root(
&self,
_barrier: &SingleWriterBackupBarrier,
destination: impl AsRef<Path>,
) -> Result<(), LocalStorageError> {
self.validate_source_set()?;
let destination = destination.as_ref();
let expected = self.source_manifest();
if destination.exists() {
return validate_restored_set(destination, &expected);
}
let parent = destination.parent().ok_or(LocalStorageError::InvalidData)?;
crate::prepare_owner_directory(parent)?;
let name = destination
.file_name()
.and_then(|name| name.to_str())
.ok_or(LocalStorageError::InvalidData)?;
let attempt = random_attempt_id()?;
let stage = parent.join(format!(".{name}.{attempt}.restore-set"));
hiroute_platform::private_fs::create_directory(&stage)?;
for (backup, name) in [
(&self.control, "control.db"),
(&self.runtime, "runtime.db"),
(&self.secrets, "secrets.db"),
] {
let database = stage.join(name);
backup.restore_to(&database)?;
write_manifest(&manifest_path(&database), backup.manifest())?;
}
write_set_manifest(&stage.join(BACKUP_SET_MANIFEST_NAME), &expected)?;
validate_restored_set(&stage, &expected)?;
hiroute_platform::durable::directory_barrier(&stage)?;
hiroute_platform::durable::rename(&stage, destination)?;
hiroute_platform::durable::directory_barrier(parent)?;
validate_restored_set(destination, &expected)
}
/// Recovery-only replacement of the sealed live directory. Publication is a directory-level
/// swap with durable phase and fixed set-owned paths, so a restart can finish the swap without
/// ever selecting files from two generations.
pub(crate) fn restore_over_live_root(
&mut self,
barrier: &SingleWriterBackupBarrier,
live_root: &Path,
) -> Result<(), LocalStorageError> {
if self.phase != BackupSetPhase::Restoring {
self.persist_phase(barrier, BackupSetPhase::Restoring)?;
}
let parent = live_root.parent().ok_or(LocalStorageError::InvalidData)?;
crate::prepare_owner_directory(parent)?;
let live_name = live_root
.file_name()
.and_then(|name| name.to_str())
.ok_or(LocalStorageError::InvalidData)?;
let ready = parent.join(format!(".{live_name}.{}.restore-ready", self.set_id));
let previous = parent.join(format!(".{live_name}.{}.pre-migration", self.set_id));
let expected = self.source_manifest();
if live_root.exists() && previous.exists() {
validate_restored_set(live_root, &expected)?;
self.persist_phase(barrier, BackupSetPhase::Restored)?;
return Ok(());
}
if !ready.exists() {
self.restore_to_new_root(barrier, &ready)?;
} else {
validate_restored_set(&ready, &expected)?;
}
if live_root.exists() {
if previous.exists() {
return Err(LocalStorageError::InvalidData);
}
hiroute_platform::durable::rename(live_root, &previous)?;
hiroute_platform::durable::directory_barrier(parent)?;
}
if !live_root.exists() {
hiroute_platform::durable::rename(&ready, live_root)?;
hiroute_platform::durable::directory_barrier(parent)?;
}
validate_restored_set(live_root, &expected)?;
self.persist_phase(barrier, BackupSetPhase::Restored)
}
fn set_manifest(&self) -> BackupSetManifest {
BackupSetManifest {
schema: BACKUP_SET_MANIFEST_SCHEMA.to_owned(),
set_id: self.set_id.clone(),
attempt_id: self.attempt_id.clone(),
phase: self.phase,
target_schema_version: self.target_schema_version,
watermark: self.watermark.clone(),
control: self.control.manifest.clone(),
runtime: self.runtime.manifest.clone(),
secrets: self.secrets.manifest.clone(),
}
}
fn source_manifest(&self) -> BackupSetManifest {
let mut manifest = self.set_manifest();
manifest.phase = BackupSetPhase::SourcePublished;
manifest
}
fn sources_match_connections(
&self,
control: &Connection,
runtime: &Connection,
secrets: &Connection,
secret_key_id: Option<&CanonicalDigest>,
) -> Result<bool, LocalStorageError> {
Ok(
self.control.manifest.source_state_digest == database_state_digest(control)?
&& self.runtime.manifest.source_state_digest == database_state_digest(runtime)?
&& self.secrets.manifest.source_state_digest == database_state_digest(secrets)?
&& self.secrets.manifest.key_id.as_deref()
== secret_key_id.map(CanonicalDigest::as_str),
)
}
fn validate_source_set(&self) -> Result<(), LocalStorageError> {
let control = SqliteBackup::open(&self.control.path)?;
let runtime = SqliteBackup::open(&self.runtime.path)?;
let secrets = SqliteBackup::open(&self.secrets.path)?;
if control.manifest != self.control.manifest
|| runtime.manifest != self.runtime.manifest
|| secrets.manifest != self.secrets.manifest
|| control.manifest.journal_watermark != self.watermark
|| runtime.manifest.journal_watermark != self.watermark
|| secrets.manifest.journal_watermark != self.watermark
{
return Err(LocalStorageError::InvalidData);
}
Ok(())
}
}
fn validate_restored_set(
root: &Path,
expected: &BackupSetManifest,
) -> Result<(), LocalStorageError> {
validate_set_manifest_shape(expected)?;
let manifest_path = root.join(BACKUP_SET_MANIFEST_NAME);
validate_owner_file(&manifest_path)?;
let manifest: BackupSetManifest = serde_json::from_slice(&fs::read(manifest_path)?)
.map_err(|_| LocalStorageError::InvalidData)?;
if &manifest != expected || validate_set_manifest_shape(&manifest).is_err() {
return Err(LocalStorageError::InvalidData);
}
for (name, expected_manifest) in [
("control.db", &manifest.control),
("runtime.db", &manifest.runtime),
("secrets.db", &manifest.secrets),
] {
let restored = SqliteBackup::open(root.join(name))?;
if restored.manifest() != expected_manifest {
return Err(LocalStorageError::InvalidData);
}
}
Ok(())
}
fn validate_set_manifest_shape(manifest: &BackupSetManifest) -> Result<(), LocalStorageError> {
let valid_id = |value: &str| {
!value.is_empty()
&& value.len() <= 96
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-'))
};
if manifest.schema != BACKUP_SET_MANIFEST_SCHEMA
|| !valid_id(&manifest.set_id)
|| !valid_id(&manifest.attempt_id)
|| manifest.target_schema_version == 0
|| manifest.watermark.is_empty()
|| manifest.control.database_kind != "control"
|| manifest.runtime.database_kind != "runtime"
|| manifest.secrets.database_kind != "secrets"
|| manifest.control.journal_watermark != manifest.watermark
|| manifest.runtime.journal_watermark != manifest.watermark
|| manifest.secrets.journal_watermark != manifest.watermark
{
return Err(LocalStorageError::InvalidData);
}
Ok(())
}
fn validate_manifest_against_backups(
root: &Path,
manifest: &BackupSetManifest,
) -> Result<(SqliteBackup, SqliteBackup, SqliteBackup), LocalStorageError> {
let control = SqliteBackup::open(root.join("control.db"))?;
let runtime = SqliteBackup::open(root.join("runtime.db"))?;
let secrets = SqliteBackup::open(root.join("secrets.db"))?;
if control.manifest != manifest.control
|| runtime.manifest != manifest.runtime
|| secrets.manifest != manifest.secrets
{
return Err(LocalStorageError::InvalidData);
}
Ok((control, runtime, secrets))
}
fn valid_phase_transition(from: BackupSetPhase, to: BackupSetPhase) -> bool {
matches!(
(from, to),
(
BackupSetPhase::SourcePublished,
BackupSetPhase::ControlMigrated
) | (
BackupSetPhase::ControlMigrated,
BackupSetPhase::RuntimeMigrated
) | (
BackupSetPhase::RuntimeMigrated,
BackupSetPhase::SecretsMigrated
) | (BackupSetPhase::SecretsMigrated, BackupSetPhase::Completed)
| (
BackupSetPhase::SourcePublished
| BackupSetPhase::ControlMigrated
| BackupSetPhase::RuntimeMigrated
| BackupSetPhase::SecretsMigrated,
BackupSetPhase::Restoring
)
| (BackupSetPhase::Restoring, BackupSetPhase::Restored)
)
}
struct DatabaseIdentity {
kind: String,
store_uuid: String,
schema_version: u32,
key_id: Option<String>,
}
pub(crate) fn database_state_digest(
connection: &Connection,
) -> Result<CanonicalDigest, LocalStorageError> {
let user_version: u32 = connection.query_row("PRAGMA user_version", [], |row| row.get(0))?;
let mut hasher = Sha256::new();
digest_field(&mut hasher, b"hiroute.sqlite-logical-state/v1");
digest_field(&mut hasher, &user_version.to_le_bytes());
let schema = {
let mut statement = connection.prepare(
"SELECT type, name, coalesce(sql, '') FROM sqlite_schema
WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name",
)?;
statement
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
))
})?
.collect::<Result<Vec<_>, _>>()?
};
for (kind, name, sql) in &schema {
digest_field(&mut hasher, kind.as_bytes());
digest_field(&mut hasher, name.as_bytes());
digest_field(&mut hasher, sql.as_bytes());
}
for (_, table, _) in schema.iter().filter(|(kind, _, _)| kind == "table") {
let columns = {
let mut statement =
connection.prepare("SELECT name FROM pragma_table_info(?1) ORDER BY cid")?;
statement
.query_map([table], |row| row.get::<_, String>(0))?
.collect::<Result<Vec<_>, _>>()?
};
if columns.is_empty() {
return Err(LocalStorageError::InvalidData);
}
digest_field(&mut hasher, table.as_bytes());
for column in &columns {
digest_field(&mut hasher, column.as_bytes());
}
let projection = columns
.iter()
.map(|column| quote_identifier(column))
.collect::<Vec<_>>()
.join(", ");
let query = format!(
"SELECT {projection} FROM {} ORDER BY {projection}",
quote_identifier(table)
);
let mut statement = connection.prepare(&query)?;
let column_count = statement.column_count();
let mut rows = statement.query([])?;
while let Some(row) = rows.next()? {
digest_field(&mut hasher, b"row");
for index in 0..column_count {
match row.get_ref(index)? {
ValueRef::Null => digest_field(&mut hasher, b"null"),
ValueRef::Integer(value) => {
digest_field(&mut hasher, b"integer");
digest_field(&mut hasher, &value.to_le_bytes());
}
ValueRef::Real(value) => {
digest_field(&mut hasher, b"real");
digest_field(&mut hasher, &value.to_bits().to_le_bytes());
}
ValueRef::Text(value) => {
digest_field(&mut hasher, b"text");
digest_field(&mut hasher, value);
}
ValueRef::Blob(value) => {
digest_field(&mut hasher, b"blob");
digest_field(&mut hasher, value);
}
}
}
}
}
CanonicalDigest::parse(format!("sha256:{}", encode_hex(&hasher.finalize())))
.map_err(|_| LocalStorageError::InvalidData)
}
fn digest_field(hasher: &mut Sha256, value: &[u8]) {
hasher.update((value.len() as u64).to_le_bytes());
hasher.update(value);
}
fn quote_identifier(value: &str) -> String {
format!("\"{}\"", value.replace('"', "\"\""))
}
fn database_identity(connection: &Connection) -> Result<DatabaseIdentity, LocalStorageError> {
let schema_version = connection.query_row("PRAGMA user_version", [], |row| row.get(0))?;
let table_exists = |name: &str| -> Result<bool, LocalStorageError> {
Ok(connection.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
params![name],
|row| row.get(0),
)?)
};
if table_exists("operations")? {
let (store_uuid, key_id) = generic_store_binding(connection)?;
Ok(DatabaseIdentity {
kind: "control".to_owned(),
store_uuid,
schema_version,
key_id,
})
} else if table_exists("runtime_state")? {
let (store_uuid, key_id) = generic_store_binding(connection)?;
Ok(DatabaseIdentity {
kind: "runtime".to_owned(),
store_uuid,
schema_version,
key_id,
})
} else if table_exists("secret_store_meta")? {
let binding = if schema_version >= 3 {
connection
.query_row(
"SELECT store_uuid, key_id FROM secret_store_meta WHERE singleton = 1",
[],
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
)
.optional()?
} else {
None
};
Ok(DatabaseIdentity {
kind: "secrets".to_owned(),
store_uuid: binding
.as_ref()
.map(|binding| binding.0.clone())
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "legacy-unbound".to_owned()),
schema_version,
key_id: binding
.map(|binding| binding.1)
.filter(|value| !value.is_empty()),
})
} else {
Err(LocalStorageError::InvalidData)
}
}
/// A manually restored, completed source may have new business rows. Match its identity,
/// rather than the old state digest; interrupted migration still uses its exact source digest.
pub(crate) fn validate_restored_source_identity(
connection: &Connection,
source: &BackupManifest,
) -> Result<(), LocalStorageError> {
let integrity: String = connection.query_row("PRAGMA integrity_check", [], |row| row.get(0))?;
let identity = database_identity(connection)?;
if integrity != "ok"
|| source.store_uuid.is_empty()
|| source.store_uuid == "legacy-unbound"
|| identity.kind != source.database_kind
|| identity.store_uuid != source.store_uuid
|| identity.schema_version != source.schema_version
|| identity.key_id != source.key_id
{
return Err(LocalStorageError::InvalidData);
}
Ok(())
}
fn generic_store_binding(
connection: &Connection,
) -> Result<(String, Option<String>), LocalStorageError> {
let exists: bool = connection.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'storage_meta')",
[],
|row| row.get(0),
)?;
if !exists {
return Ok(("legacy-unbound".to_owned(), None));
}
Ok(connection
.query_row(
"SELECT store_uuid, key_id FROM storage_meta WHERE singleton = 1",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.optional()?
.unwrap_or_else(|| ("legacy-unbound".to_owned(), None)))
}
fn standalone_watermark(connection: &Connection) -> Result<String, LocalStorageError> {
let identity = database_identity(connection)?;
if identity.kind == "control" {
control_watermark(connection)
} else {
Ok(format!(
"standalone:{}:{}",
identity.kind, identity.schema_version
))
}
}
fn control_watermark(connection: &Connection) -> Result<String, LocalStorageError> {
let (count, generation): (u64, u64) = connection.query_row(
"SELECT count(*), coalesce(max(generation), 0) FROM operations",
[],
|row| Ok((row.get(0)?, row.get(1)?)),
)?;
Ok(format!("control:{count}:{generation}"))
}
fn manifest_exact_except_attempt(left: &BackupManifest, right: &BackupManifest) -> bool {
left.schema == right.schema
&& left.database_kind == right.database_kind
&& left.store_uuid == right.store_uuid
&& left.schema_version == right.schema_version
&& left.source_digest == right.source_digest
&& left.source_state_digest == right.source_state_digest
&& left.key_id == right.key_id
&& left.journal_watermark == right.journal_watermark
}
fn manifest_path(path: &Path) -> PathBuf {
let mut value = path.as_os_str().to_os_string();
value.push(".manifest.json");
PathBuf::from(value)
}
fn set_manifest_path(root: &Path) -> PathBuf {
root.join(BACKUP_SET_MANIFEST_NAME)
}
fn set_manifest_stage_prefix() -> &'static str {
".backup-set.manifest."
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum SetManifestStageKind {
Writing,
Committed,
}
fn set_manifest_stages(
root: &Path,
) -> Result<Vec<(PathBuf, SetManifestStageKind)>, LocalStorageError> {
let mut stages = Vec::new();
for entry in fs::read_dir(root)? {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
if !name.starts_with(set_manifest_stage_prefix()) || !name.ends_with(".stage.json") {
continue;
}
let remainder = &name[set_manifest_stage_prefix().len()..];
let (attempt, kind) = if let Some(attempt) = remainder.strip_suffix(".committed.stage.json")
{
(attempt, SetManifestStageKind::Committed)
} else if let Some(attempt) = remainder.strip_suffix(".writing.stage.json") {
(attempt, SetManifestStageKind::Writing)
} else if let Some(attempt) = remainder.strip_suffix(".stage.json") {
// Legacy stages were visible from file creation through fsync and therefore carry no
// proof that their bytes were ever committed. Never promote them, even when parsing
// happens to succeed.
(attempt, SetManifestStageKind::Writing)
} else {
return Err(LocalStorageError::InvalidData);
};
if attempt.is_empty()
|| attempt.len() > 96
|| !attempt
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-'))
{
return Err(LocalStorageError::InvalidData);
}
stages.push((entry.path(), kind));
}
Ok(stages)
}
fn publish_set_manifest(
root: &Path,
manifest: &BackupSetManifest,
replace: bool,
) -> Result<(), LocalStorageError> {
validate_set_manifest_shape(manifest)?;
let attempt = random_attempt_id()?;
let writing = root.join(format!(
"{}{attempt}.writing.stage.json",
set_manifest_stage_prefix()
));
let committed = root.join(format!(
"{}{attempt}.committed.stage.json",
set_manifest_stage_prefix()
));
write_set_manifest(&writing, manifest)?;
// The committed name is the durable proof that the complete JSON bytes were fsynced. A
// writing name can therefore always be discarded after a crash without attempting to parse
// bytes that may be empty or truncated.
hiroute_platform::durable::directory_barrier(root)?;
hiroute_platform::durable::rename(&writing, &committed)?;
hiroute_platform::durable::directory_barrier(root)?;
let final_path = set_manifest_path(root);
if !replace && final_path.exists() {
return Err(LocalStorageError::InvalidData);
}
hiroute_platform::durable::rename(&committed, &final_path)?;
hiroute_platform::durable::directory_barrier(root)?;
validate_owner_file(&final_path)
}
fn reconcile_set_manifest_publication(root: &Path) -> Result<bool, LocalStorageError> {
let final_path = set_manifest_path(root);
let stages = set_manifest_stages(root)?;
if final_path.exists() {
validate_owner_file(&final_path)?;
for (stage, _) in stages {
tighten_and_remove_owner_orphan(&stage)?;
}
hiroute_platform::durable::directory_barrier(root)?;
return Ok(true);
}
let mut committed = Vec::new();
for (stage, kind) in stages {
match kind {
SetManifestStageKind::Writing => tighten_and_remove_owner_orphan(&stage)?,
SetManifestStageKind::Committed => committed.push(stage),
}
}
if committed.is_empty() {
hiroute_platform::durable::directory_barrier(root)?;
return Ok(false);
}
if committed.len() != 1 {
return Err(LocalStorageError::InvalidData);
}
let stage = committed.pop().expect("length checked");
validate_owner_file(&stage)?;
let manifest: BackupSetManifest =
serde_json::from_slice(&fs::read(&stage)?).map_err(|_| LocalStorageError::InvalidData)?;
validate_set_manifest_shape(&manifest)?;
validate_manifest_against_backups(root, &manifest)?;
hiroute_platform::durable::rename(stage, &final_path)?;
hiroute_platform::durable::directory_barrier(root)?;
validate_owner_file(&final_path)?;
Ok(true)
}
fn write_manifest(path: &Path, manifest: &BackupManifest) -> Result<(), LocalStorageError> {
let bytes = serde_json::to_vec(manifest).map_err(|_| LocalStorageError::InvalidData)?;
write_owner_file(path, &bytes)
}
fn write_set_manifest(path: &Path, manifest: &BackupSetManifest) -> Result<(), LocalStorageError> {
let bytes = serde_json::to_vec(manifest).map_err(|_| LocalStorageError::InvalidData)?;
write_owner_file(path, &bytes)
}
fn write_owner_file(path: &Path, bytes: &[u8]) -> Result<(), LocalStorageError> {
hiroute_platform::durable::write_private_new(path, bytes)?;
validate_owner_file(path)
}
fn file_digest(path: &Path) -> Result<CanonicalDigest, LocalStorageError> {
let mut file = fs::File::open(path)?;
let before = file.metadata()?;
if !before.is_file() {
return Err(LocalStorageError::InvalidData);
}
let digest = stream_digest(&mut file, before.len())?;
let after = file.metadata()?;
if before.len() != after.len() || before.modified()? != after.modified()? {
return Err(LocalStorageError::InvalidData);
}
Ok(digest)
}
// Preserve the released raw-byte SHA-256 while bounding memory and the admitted read length.
// A stopped source that shrinks or grows cannot become a verified recovery source.
fn stream_digest(
reader: &mut impl std::io::Read,
byte_count: u64,
) -> Result<CanonicalDigest, LocalStorageError> {
let mut remaining = byte_count;
let mut buffer = zeroize::Zeroizing::new([0_u8; 64 * 1024]);
let mut hasher = Sha256::new();
while remaining != 0 {
let count = remaining.min(buffer.len() as u64) as usize;
reader.read_exact(&mut buffer[..count])?;
hasher.update(&buffer[..count]);
remaining -= count as u64;
}
loop {
match reader.read(&mut buffer[..1]) {
Ok(0) => break,
Ok(_) => return Err(LocalStorageError::InvalidData),
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(error) => return Err(error.into()),
}
}
CanonicalDigest::parse(format!("sha256:{}", encode_hex(&hasher.finalize())))
.map_err(|_| LocalStorageError::InvalidData)
}
fn random_attempt_id() -> Result<String, LocalStorageError> {
let mut bytes = [0_u8; 12];
getrandom::fill(&mut bytes).map_err(|_| LocalStorageError::Crypto)?;
Ok(encode_hex(&bytes))
}
fn encode_hex(bytes: &[u8]) -> String {
use std::fmt::Write as _;
let mut encoded = String::with_capacity(bytes.len() * 2);
for byte in bytes {
write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail");
}
encoded
}
fn cleanup_orphan_stages(parent: &Path, file_name: &str) -> Result<(), LocalStorageError> {
let prefix = format!(".{file_name}.");
for entry in fs::read_dir(parent)? {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
if name.starts_with(&prefix) && name.contains(".stage.") {
tighten_and_remove_owner_orphan(&entry.path())?;
}
}
hiroute_platform::durable::directory_barrier(parent)?;
Ok(())
}
fn tighten_and_remove_owner_orphan(path: &Path) -> Result<(), LocalStorageError> {
let metadata = fs::symlink_metadata(path)?;
if metadata.file_type().is_symlink() || !metadata.file_type().is_file() {
return Err(LocalStorageError::Permission);
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if metadata.nlink() != 1 {
return Err(LocalStorageError::Permission);
}
}
validate_owner_file(path)?;
fs::remove_file(path)?;
Ok(())
}
fn validate_database(path: &Path) -> Result<(), LocalStorageError> {
let connection = Connection::open_with_flags(
path,
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
let result: String = connection.query_row("PRAGMA integrity_check", [], |row| row.get(0))?;
if result == "ok" {
Ok(())
} else {
Err(LocalStorageError::InvalidData)
}
}
fn set_owner_file_permissions(path: &Path) -> Result<(), LocalStorageError> {
#[cfg(windows)]
hiroute_platform::windows::security::verify_owner_only(path, false)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(path, fs::Permissions::from_mode(0o600))?;
}
validate_owner_file(path)
}
fn validate_owner_file(path: &Path) -> Result<(), LocalStorageError> {
#[cfg(windows)]
let _held = hiroute_platform::windows::security::open_owner_only(path, false)?;
let metadata = fs::symlink_metadata(path)?;
if metadata.file_type().is_symlink() || !metadata.file_type().is_file() {
return Err(LocalStorageError::Permission);
}
Ok(())
}
#[cfg(test)]
mod tests {
use crate::test_tempdir as tempdir;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use super::*;
use crate::{ControlStore, LocalSecretStore, RuntimeStore};
#[test]
fn unique_stage_and_stale_final_do_not_block_exact_publication_and_restore() {
let directory = tempdir().unwrap();
let root = directory.path().join("data");
let control = ControlStore::open(
&crate::test_storage_authority(),
root.join("control.db"),
root.join("migration"),
)
.unwrap();
let backup_path = root.join("backups/control.db");
let first = control
.with_connection(|connection| {
SqliteBackup::create(&crate::test_storage_authority(), connection, &backup_path)
})
.unwrap();
fs::write(manifest_path(&backup_path), b"stale-final").unwrap();
let parent = backup_path.parent().unwrap();
let orphan = parent.join(".control.db.deadbeef.stage.db");
#[cfg(windows)]
hiroute_platform::durable::write_private_new(&orphan, b"orphan").unwrap();
#[cfg(not(windows))]
fs::write(&orphan, b"orphan").unwrap();
set_owner_file_permissions(&orphan).unwrap();
let second = control
.with_connection(|connection| {
SqliteBackup::create(&crate::test_storage_authority(), connection, &backup_path)
})
.unwrap();
assert!(second.manifest().attempt_id != first.manifest().attempt_id);
assert!(!orphan.exists());
let restored = root.join("restored/control.db");
second.restore_to(&restored).unwrap();
validate_database(&restored).unwrap();
#[cfg(windows)]
for path in [&backup_path, &manifest_path(&backup_path), &restored] {
hiroute_platform::windows::security::verify_owner_only(path, false).unwrap();
}
}
#[test]
fn half_published_database_or_manifest_is_reconciled_by_a_fresh_attempt() {
let directory = tempdir().unwrap();
let root = directory.path().join("data");
let control = ControlStore::open(
&crate::test_storage_authority(),
root.join("control.db"),
root.join("migration"),
)
.unwrap();
let backup_path = root.join("backups/control.db");
control
.with_connection(|connection| {
SqliteBackup::create(&crate::test_storage_authority(), connection, &backup_path)
})
.unwrap();
fs::remove_file(manifest_path(&backup_path)).unwrap();
let recovered_database_only = control
.with_connection(|connection| {
SqliteBackup::create(&crate::test_storage_authority(), connection, &backup_path)
})
.unwrap();
SqliteBackup::open(recovered_database_only.path()).unwrap();
fs::remove_file(&backup_path).unwrap();
let recovered_manifest_only = control
.with_connection(|connection| {
SqliteBackup::create(&crate::test_storage_authority(), connection, &backup_path)
})
.unwrap();
SqliteBackup::open(recovered_manifest_only.path()).unwrap();
}
#[cfg(unix)]
#[test]
fn vacuum_created_0644_stage_is_tightened_and_reconciled_across_two_restarts() {
use std::os::unix::fs::MetadataExt;
let directory = tempdir().unwrap();
let root = directory.path().join("data");
let control = ControlStore::open(
&crate::test_storage_authority(),
root.join("control.db"),
root.join("migration"),
)
.unwrap();
control.with_connection(|connection| {
connection
.execute(
"INSERT INTO dependency_revisions(workspace_id, dependency_key, revision)
VALUES ('personal/default', 'prices', 9)",
[],
)
.unwrap();
});
let backup_path = root.join("backups/control.db");
let parent = backup_path.parent().unwrap();
crate::prepare_owner_directory(parent).unwrap();
for crash_id in ["after-vacuum-one", "after-vacuum-two"] {
let orphan = parent.join(format!(".control.db.{crash_id}.stage.db"));
control.with_connection(|connection| {
connection
.execute_batch("PRAGMA wal_checkpoint(FULL)")
.unwrap();
connection
.execute("VACUUM main INTO ?1", [orphan.to_string_lossy().as_ref()])
.unwrap();
});
fs::set_permissions(&orphan, fs::Permissions::from_mode(0o644)).unwrap();
let metadata = fs::symlink_metadata(&orphan).unwrap();
assert_eq!(metadata.mode() & 0o777, 0o644);
assert_eq!(metadata.uid(), rustix::process::getuid().as_raw());
let backup = control
.with_connection(|connection| {
SqliteBackup::create(&crate::test_storage_authority(), connection, &backup_path)
})
.unwrap();
assert!(!orphan.exists());
assert_eq!(
backup.manifest().source_state_digest,
control.with_connection(|connection| database_state_digest(connection).unwrap())
);
}
let reopened = SqliteBackup::open(&backup_path).unwrap();
let restored = root.join("restored/control.db");
reopened.restore_to(&restored).unwrap();
let restored = Connection::open_with_flags(
restored,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)
.unwrap();
assert_eq!(
database_state_digest(&restored).unwrap(),
reopened.manifest().source_state_digest
);
}
#[cfg(unix)]
#[test]
fn orphan_stage_cleanup_rejects_a_symlink_without_touching_its_target() {
use std::os::unix::fs::symlink;
let directory = tempdir().unwrap();
let root = directory.path().join("data");
let control = ControlStore::open(
&crate::test_storage_authority(),
root.join("control.db"),
root.join("migration"),
)
.unwrap();
let backup_path = root.join("backups/control.db");
let parent = backup_path.parent().unwrap();
crate::prepare_owner_directory(parent).unwrap();
let target = root.join("must-not-change");
fs::write(&target, b"owner-data").unwrap();
set_owner_file_permissions(&target).unwrap();
let orphan = parent.join(".control.db.symlink.stage.db");
symlink(&target, &orphan).unwrap();
assert!(
control
.with_connection(|connection| {
SqliteBackup::create(&crate::test_storage_authority(), connection, &backup_path)
})
.is_err()
);
assert!(
fs::symlink_metadata(&orphan)
.unwrap()
.file_type()
.is_symlink()
);
assert_eq!(fs::read(target).unwrap(), b"owner-data");
}
#[test]
fn backup_set_manifest_reopens_publish_boundaries_and_rejects_mismatch() {
let directory = tempdir().unwrap();
let root = directory.path().join("data");
let control = ControlStore::open(
&crate::test_storage_authority(),
root.join("control.db"),
root.join("migration"),
)
.unwrap();
let runtime = RuntimeStore::open(
&crate::test_storage_authority(),
root.join("runtime.db"),
root.join("migration"),
)
.unwrap();
let secrets = LocalSecretStore::open(
&crate::test_storage_authority(),
root.join("secrets.db"),
root.join("master-key"),
root.join("migration"),
)
.unwrap();
let backup_root = root.join("backup-set");
let set = control
.with_connection(|control_connection| {
runtime.with_connection(|runtime_connection| {
secrets.with_connection(|secret_connection| {
BackupSet::create(
&test_writer_barrier(),
&backup_root,
control_connection,
runtime_connection,
secret_connection,
)
})
})
})
.unwrap();
let set_id = set.set_id().to_owned();
drop(set);
let final_manifest = set_manifest_path(&backup_root);
let stage_manifest = backup_root.join(format!(
"{}crash-before-rename.committed.stage.json",
set_manifest_stage_prefix()
));
fs::rename(&final_manifest, &stage_manifest).unwrap();
hiroute_platform::durable::directory_barrier(&backup_root).unwrap();
let reopened = BackupSet::open(&crate::test_storage_authority(), &backup_root).unwrap();
assert_eq!(reopened.set_id(), set_id);
assert_eq!(reopened.phase(), BackupSetPhase::SourcePublished);
assert!(final_manifest.exists());
assert!(!stage_manifest.exists());
let mut staged_phase = reopened.set_manifest();
staged_phase.phase = BackupSetPhase::ControlMigrated;
let phase_stage = backup_root.join(format!(
"{}phase-before-rename.committed.stage.json",
set_manifest_stage_prefix()
));
write_set_manifest(&phase_stage, &staged_phase).unwrap();
let reopened = BackupSet::open(&crate::test_storage_authority(), &backup_root).unwrap();
assert_eq!(reopened.phase(), BackupSetPhase::SourcePublished);
assert!(!phase_stage.exists());
let original = fs::read(&final_manifest).unwrap();
let mut mismatch: BackupSetManifest = serde_json::from_slice(&original).unwrap();
mismatch.control.source_digest = CanonicalDigest::of_bytes(b"mismatched-set");
fs::write(&final_manifest, serde_json::to_vec(&mismatch).unwrap()).unwrap();
assert!(BackupSet::open(&crate::test_storage_authority(), &backup_root).is_err());
fs::write(&final_manifest, original).unwrap();
let exact = BackupSet::open(&crate::test_storage_authority(), &backup_root).unwrap();
let restored = root.join("restored-source-set");
exact
.restore_to_new_root(&test_writer_barrier(), &restored)
.unwrap();
#[cfg(windows)]
hiroute_platform::windows::security::verify_owner_only(&restored, true).unwrap();
for (name, backup) in [
("control.db", &exact.control),
("runtime.db", &exact.runtime),
("secrets.db", &exact.secrets),
] {
#[cfg(windows)]
{
hiroute_platform::windows::security::verify_owner_only(&restored.join(name), false)
.unwrap();
hiroute_platform::windows::security::verify_owner_only(
&manifest_path(&restored.join(name)),
false,
)
.unwrap();
}
let connection = Connection::open_with_flags(
restored.join(name),
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)
.unwrap();
assert_eq!(
database_state_digest(&connection).unwrap(),
backup.manifest().source_state_digest
);
}
#[cfg(windows)]
hiroute_platform::windows::security::verify_owner_only(
&restored.join(BACKUP_SET_MANIFEST_NAME),
false,
)
.unwrap();
}
#[test]
fn backup_set_has_one_watermark_and_refuses_an_admitted_writer() {
let directory = tempdir().unwrap();
let root = directory.path().join("data");
let control = ControlStore::open(
&crate::test_storage_authority(),
root.join("control.db"),
root.join("migration"),
)
.unwrap();
let runtime = RuntimeStore::open(
&crate::test_storage_authority(),
root.join("runtime.db"),
root.join("migration"),
)
.unwrap();
let secrets = LocalSecretStore::open(
&crate::test_storage_authority(),
root.join("secrets.db"),
root.join("master-key"),
root.join("migration"),
)
.unwrap();
let set = control
.with_connection(|control_connection| {
runtime.with_connection(|runtime_connection| {
secrets.with_connection(|secret_connection| {
BackupSet::create(
&test_writer_barrier(),
root.join("backup-set"),
control_connection,
runtime_connection,
secret_connection,
)
})
})
})
.unwrap();
assert_eq!(set.control.manifest().journal_watermark, set.watermark());
assert_eq!(set.runtime.manifest().journal_watermark, set.watermark());
assert_eq!(set.secrets.manifest().journal_watermark, set.watermark());
let restored = root.join("restored-set");
set.restore_to_new_root(&test_writer_barrier(), &restored)
.unwrap();
set.restore_to_new_root(&test_writer_barrier(), &restored)
.unwrap();
for name in ["control.db", "runtime.db", "secrets.db"] {
SqliteBackup::open(restored.join(name)).unwrap();
}
control.with_connection(|connection| {
connection
.execute(
"INSERT INTO writer_claim(singleton, operation_id, admitted_at)
VALUES (1, 'op_11111111111111111111111111111111', unixepoch())",
[],
)
.unwrap();
});
assert!(
control
.with_connection(|control_connection| {
runtime.with_connection(|runtime_connection| {
secrets.with_connection(|secret_connection| {
BackupSet::create(
&test_writer_barrier(),
root.join("blocked-set"),
control_connection,
runtime_connection,
secret_connection,
)
})
})
})
.is_err()
);
}
}