The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
refactor(core): migrate watcher state to tinystorage backend
Steven Enamakel2793691c352b
diff --git a/crates/openhuman-core/src/config/workspace/state.rs b/crates/openhuman-core/src/config/workspace/state.rsindex 61f03fc57f..5a75dfcdc5 100644--- a/crates/openhuman-core/src/config/workspace/state.rs+++ b/crates/openhuman-core/src/config/workspace/state.rs@@ -1,128 +1,171 @@-//! SQLite-backed mtime state store for the vault file watcher.+//! Machine-local mtime state for the vault file watcher.//!-//! **Deliberately local, not on the storage port.** The rows are absolute-//! paths on this machine and the mtimes the local filesystem reported for-//! them. Neither means anything on another host, so a shared backend (a-//! MongoDB behind several cores) would make one machine skip, or re-ingest,-//! files because of what a different machine saw. It stays a per-machine-//! SQLite file; `scripts/ci/check-storage-bypass.mjs` allowlists it with this-//! reason.-//!-//! Persists `path → last_mtime_secs` across restarts so the watcher can-//! detect real changes on startup without re-ingesting every file from-//! scratch. One row per tracked file; the table is created lazily on-//! first use.-//!-//! ## Schema-//!-//! ```sql-//! CREATE TABLE IF NOT EXISTS vault_watcher_state (-//! path TEXT PRIMARY KEY,-//! mtime_secs INTEGER NOT NULL,-//! deleted INTEGER NOT NULL DEFAULT 0-//! );-//! ```--use crate::storage::rusqlite;-use rusqlite::{params, Connection, OptionalExtension, Result as SqlResult};+//! The rows contain absolute paths and mtimes observed by this machine, so+//! they use a dedicated local SQLite backend opened through the storage+//! facade rather than the process-wide backend (which may be shared across+//! hosts). The legacy SQLite table is read only by the host migration path.+use std::path::{Path, PathBuf};+use std::sync::Arc;++use anyhow::{Context, Result, anyhow};+use serde::{Deserialize, Serialize};+use serde_json::Value;+use sha2::{Digest, Sha256};+use tinystoragedrivers::{Scope, StorageBackend, StorageConfig};+use tinystoragedrivers_api::KvStore;+use tinystoragedrivers_types::{Cursor, Precondition};+const KEY_PREFIX: &str = "openhuman.vault-watcher.";+const PAGE_SIZE: usize = 256;++#[derive(Clone)]pub struct WatcherStateStore {- conn: Connection,+ kv: Arc<dyn KvStore>,}-#[derive(Debug, Clone)]+#[derive(Debug, Clone, PartialEq, Eq)]pub struct FileState {pub path: PathBuf,pub mtime_secs: u64,pub deleted: bool,}+#[derive(Debug, Clone, Serialize, Deserialize)]+struct StoredFileState {+ path: String,+ mtime_secs: u64,+ deleted: bool,+}+impl WatcherStateStore {- /// Open (or create) the state database at `db_path`.- pub fn open(db_path: &Path) -> SqlResult<Self> {- log::debug!("[watcher::state] open db_path={}", db_path.display());- let conn = Connection::open(db_path)?;- conn.execute_batch(- "PRAGMA journal_mode = WAL;- PRAGMA synchronous = NORMAL;- CREATE TABLE IF NOT EXISTS vault_watcher_state (- path TEXT PRIMARY KEY,- mtime_secs INTEGER NOT NULL,- deleted INTEGER NOT NULL DEFAULT 0- );",- )?;- log::debug!("[watcher::state] open ok");- Ok(Self { conn })+ /// Open the machine-local store at `db_path` through the TSD facade.+ pub fn open(db_path: &Path) -> Result<Self> {+ let url = format!("sqlite:{}", db_path.display());+ let config = StorageConfig::parse(&url).context("parse watcher storage URL")?;+ let backend = crate::storage::block_on(tinystoragedrivers::open(&config))+ .context("open watcher storage backend")?;+ Self::from_backend(backend)}- /// Upsert the mtime for `path`, marking it as not-deleted.- pub fn record_seen(&mut self, path: &Path, mtime_secs: u64) -> SqlResult<()> {- log::trace!(- "[watcher::state] record_seen file={} mtime={}",- path.file_name().unwrap_or_default().to_string_lossy(),- mtime_secs- );- self.conn.execute(- "INSERT INTO vault_watcher_state (path, mtime_secs, deleted)- VALUES (?1, ?2, 0)- ON CONFLICT(path) DO UPDATE SET mtime_secs = ?2, deleted = 0",- params![path.to_string_lossy().as_ref(), mtime_secs as i64],- )?;+ fn from_backend(backend: Arc<dyn StorageBackend>) -> Result<Self> {+ let scope = Scope::new("local/vault-watcher").context("build watcher scope")?;+ let storage = backend+ .for_scope(&scope)+ .map_err(anyhow::Error::from)+ .context("open watcher scope")?;+ Ok(Self {+ kv: Arc::clone(storage.kv()),+ })+ }++ /// Upsert the observed mtime for `path`, marking it as present.+ pub fn record_seen(&mut self, path: &Path, mtime_secs: u64) -> Result<()> {+ let path = path.to_string_lossy().into_owned();+ let key = key_for(&path);+ let value = serde_json::to_value(StoredFileState {+ path,+ mtime_secs,+ deleted: false,+ })?;+ let kv = Arc::clone(&self.kv);+ crate::storage::block_on(async move {+ kv.put(&key, value).await.map(|_| ())+ })+ .context("write watcher state")?;Ok(())}- /// Mark `path` as deleted (keeps the row so we don't re-process on restart).- pub fn record_deleted(&mut self, path: &Path) -> SqlResult<()> {- log::trace!(- "[watcher::state] record_deleted file={}",- path.file_name().unwrap_or_default().to_string_lossy()- );- self.conn.execute(- "INSERT INTO vault_watcher_state (path, mtime_secs, deleted)- VALUES (?1, 0, 1)- ON CONFLICT(path) DO UPDATE SET deleted = 1",- params![path.to_string_lossy().as_ref()],- )?;+ /// Mark `path` deleted, retaining its prior mtime if one exists.+ pub fn record_deleted(&mut self, path: &Path) -> Result<()> {+ let path = path.to_string_lossy().into_owned();+ let key = key_for(&path);+ let kv = Arc::clone(&self.kv);+ crate::storage::block_on(async move {+ loop {+ let current = kv.get(&key).await?;+ let (mtime_secs, expected) = match current {+ Some(current) => {+ let state = decode_state(¤t.doc)?;+ ensure_path(&state, &path)?;+ (state.mtime_secs, current.unchanged())+ }+ None => (0, Precondition::Absent),+ };+ let value = serde_json::to_value(StoredFileState {+ path: path.clone(),+ mtime_secs,+ deleted: true,+ })?;+ match kv.cas(&key, value, expected).await {+ Ok(_) => return Ok(()),+ Err(error) if error.kind() == tinystoragedrivers_types::ErrorKind::Conflict => {+ continue;+ }+ Err(error) => return Err(error),+ }+ }+ })+ .context("write deleted watcher state")?;Ok(())}- /// Return the stored mtime for `path`, or `None` if never seen / deleted.- pub fn last_mtime(&self, path: &Path) -> SqlResult<Option<u64>> {- let mut stmt = self.conn.prepare_cached(- "SELECT mtime_secs, deleted FROM vault_watcher_state WHERE path = ?1",- )?;- let result = stmt- .query_row(params![path.to_string_lossy().as_ref()], |row| {- let mtime: i64 = row.get(0)?;- let deleted: bool = row.get(1)?;- Ok((mtime as u64, deleted))- })- .optional()?;-- Ok(result.and_then(|(mtime, deleted)| (!deleted).then_some(mtime)))+ /// Return the stored mtime for `path`, or `None` if absent or deleted.+ pub fn last_mtime(&self, path: &Path) -> Result<Option<u64>> {+ let path = path.to_string_lossy().into_owned();+ let key = key_for(&path);+ let kv = Arc::clone(&self.kv);+ let value = crate::storage::block_on(async move { kv.get(&key).await })+ .context("read watcher state")?;+ let Some(value) = value else {+ return Ok(None);+ };+ let state = decode_state(&value.doc)?;+ ensure_path(&state, &path)?;+ Ok((!state.deleted).then_some(state.mtime_secs))+ }++ /// Load every non-deleted state in stable key order.+ pub fn load_all(&self) -> Result<Vec<FileState>> {+ let kv = Arc::clone(&self.kv);+ crate::storage::block_on(async move {+ let mut cursor: Option<Cursor> = None;+ let mut states = Vec::new();+ loop {+ let page = kv.list(KEY_PREFIX, cursor, PAGE_SIZE).await?;+ for item in page.items {+ let state = decode_state(&item.doc)?;+ if !state.deleted {+ states.push(FileState {+ path: PathBuf::from(state.path),+ mtime_secs: state.mtime_secs,+ deleted: false,+ });+ }+ }+ let Some(next) = page.next else { break };+ cursor = Some(next);+ }+ Ok(states)+ })+ .context("list watcher state")}+}++fn key_for(path: &str) -> String {+ let digest = Sha256::digest(path.as_bytes());+ format!("{KEY_PREFIX}{}", hex::encode(digest))+}++fn decode_state(value: &Value) -> Result<StoredFileState> {+ serde_json::from_value(value.clone()).context("decode watcher state")+}- /// Load all non-deleted states — used at startup to seed the in-memory map.- pub fn load_all(&self) -> SqlResult<Vec<FileState>> {- let mut stmt = self.conn.prepare(- "SELECT path, mtime_secs, deleted FROM vault_watcher_state WHERE deleted = 0",- )?;- let rows = stmt- .query_map([], |row| {- let path: String = row.get(0)?;- let mtime: i64 = row.get(1)?;- let deleted: bool = row.get(2)?;- Ok(FileState {- path: PathBuf::from(path),- mtime_secs: mtime as u64,- deleted,- })- })?- .collect::<SqlResult<Vec<_>>>()?;- log::debug!("[watcher::state] load_all rows={}", rows.len());- Ok(rows)+fn ensure_path(state: &StoredFileState, path: &str) -> Result<()> {+ if state.path != path {+ return Err(anyhow!("watcher state key identity mismatch"));}+ Ok(())}#[cfg(test)]