YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(db): keep macOS WAL index out of APFS mappings (#5819)
Louis Beaumontc679e4eea539
diff --git a/crates/screenpipe-connect/src/remote_sync.rs b/crates/screenpipe-connect/src/remote_sync.rsindex 3fdace91d..acf7aacc4 100644--- a/crates/screenpipe-connect/src/remote_sync.rs+++ b/crates/screenpipe-connect/src/remote_sync.rs@@ -627,16 +627,25 @@ pub async fn sync_to_remote_with_opts(/// Open a short-lived sqlx pool against the live DB and run `VACUUM INTO`/// to produce a self-contained snapshot at `dest`. SQLite's WAL mode lets-/// our connection coexist with the engine's writer pool; readers don't block-/// writers and vice versa. The pool is dropped (connections closed) before-/// we return — important because we then SFTP the snapshot file.+/// our connection coexist with the engine's writer pool; on macOS it joins+/// the recorder's unix-excl domain and process-local WAL index. The pool is+/// dropped before we return — important because we then SFTP the snapshot.async fn snapshot_db(live_db: &Path, dest: &Path) -> Result<()> {let url = format!("sqlite://{}", live_db.to_string_lossy());- let opts = SqliteConnectOptions::from_str(&url)+ let mut opts = SqliteConnectOptions::from_str(&url).with_context(|| format!("invalid sqlite url: {}", url))?- .read_only(true)- // No journal file is created for read-only opens, but be explicit:.create_if_missing(false);+ #[cfg(not(target_os = "macos"))]+ {+ opts = opts.read_only(true);+ }+ // The recorder uses unix-excl on macOS so its WAL index lives in process+ // memory instead of an APFS-backed `-shm` mapping. Snapshot connections+ // run in that same process and must join the same VFS/locking domain.+ #[cfg(target_os = "macos")]+ {+ opts = opts.vfs("unix-excl");+ }let pool = SqlitePoolOptions::new().max_connections(1)@@ -1114,6 +1123,64 @@ async fn discover_tailscale() -> Vec<DiscoveredHost> {mod tests {use super::*;+ #[tokio::test]+ async fn snapshot_reads_the_live_wal_without_a_file_backed_index_on_macos() {+ let dir = tempfile::tempdir().expect("temp dir");+ let live = dir.path().join("live.sqlite");+ let snapshot = dir.path().join("snapshot.sqlite");+ let mut live_options = SqliteConnectOptions::new()+ .filename(&live)+ .create_if_missing(true);+ #[cfg(target_os = "macos")]+ {+ live_options = live_options.vfs("unix-excl");+ }+ let live_pool = SqlitePoolOptions::new()+ .max_connections(2)+ .min_connections(2)+ .connect_with(live_options)+ .await+ .expect("open live pool");+ sqlx::query("PRAGMA journal_mode=WAL")+ .execute(&live_pool)+ .await+ .expect("enable WAL");+ sqlx::query("CREATE TABLE capture_probe (id INTEGER PRIMARY KEY, payload BLOB NOT NULL)")+ .execute(&live_pool)+ .await+ .expect("create probe");+ sqlx::query(+ "WITH RECURSIVE rows(id) AS (VALUES(1) UNION ALL SELECT id+1 FROM rows WHERE id<256) \+ INSERT INTO capture_probe(id, payload) SELECT id, randomblob(2048) FROM rows",+ )+ .execute(&live_pool)+ .await+ .expect("seed live WAL");++ snapshot_db(&live, &snapshot)+ .await+ .expect("snapshot live database");+ let snapshot_url = format!("sqlite://{}?mode=ro", snapshot.to_string_lossy());+ let snapshot_pool = SqlitePoolOptions::new()+ .max_connections(1)+ .connect(&snapshot_url)+ .await+ .expect("open snapshot");+ let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM capture_probe")+ .fetch_one(&snapshot_pool)+ .await+ .expect("read snapshot");+ assert_eq!(count, 256);++ #[cfg(target_os = "macos")]+ assert!(+ !live.with_file_name("live.sqlite-shm").exists(),+ "unix-excl snapshotting must not create a file-backed WAL index"+ );+ snapshot_pool.close().await;+ live_pool.close().await;+ }+#[test]fn test_parse_ssh_config() {let config = "Host myserver\n HostName 1.2.3.4\n User root\n Port 2222\n IdentityFile ~/.ssh/id\n\nHost *.wild\n User x\n\nHost vps\n HostName 5.6.7.8\n User admin\n";diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex 22d1d4b8e..3f5358aca 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -198,11 +198,24 @@ impl DatabaseManager {// cache_size + mmap_size are tier-configurable and applied here; the// WAL-safety pragmas that MUST be identical on every connection over this// file come from the single source of truth `WAL_SAFETY_PRAGMAS`.+ let is_in_memory =+ database_path.contains(":memory:") || database_path.contains("mode=memory");let mut connect_options: SqliteConnectOptions = connection_string.parse::<SqliteConnectOptions>()?.busy_timeout(Duration::from_secs(5)).pragma("cache_size", format!("-{}", config.cache_size_kb)).pragma("mmap_size", config.mmap_size.to_string());+ // macOS delivers an uncatchable SIGBUS when APFS invalidates a page in+ // SQLite's file-backed WAL index after the `-shm` file was shortened.+ // `unix-excl` keeps the WAL index in process memory and takes one OS+ // lock for the recorder process, while SQLite still coordinates all+ // SQLx connections internally. Screenpipe already requires exclusive+ // ownership for live capture, recovery, and maintenance operations.+ #[cfg(target_os = "macos")]+ if !is_in_memory {+ connect_options = connect_options.vfs("unix-excl");+ info!("macOS capture database using unix-excl VFS with a process-local WAL index");+ }for (pragma, value) in screenpipe_config::WAL_SAFETY_PRAGMAS {connect_options = connect_options.pragma(pragma, value);}@@ -232,20 +245,23 @@ impl DatabaseManager {.map_err(|error| quarantine_startup_error(database_file, error))?;}- // File-backed query connections have two independent write barriers:- // SQLite opens the file with mode=ro, and query_only rejects mutations- // at the connection level. In-memory databases cannot use mode=ro and- // remain writable for the many isolated test fixtures that seed them- // directly; production databases are always file-backed.- let is_in_memory =- database_path.contains(":memory:") || database_path.contains("mode=memory");+ // Every file-backed query connection enables query_only. Non-macOS+ // builds also open the file with mode=ro as an independent physical+ // barrier. macOS keeps all unix-excl handles physically RW so they+ // share one process-local WAL index instead of reopening `-shm`.+ // In-memory test databases remain writable for fixtures that seed them+ // directly.let read_connect_options = if is_in_memory {connect_options.clone()} else {- connect_options- .clone()- .read_only(true)- .pragma("query_only", "ON")+ let options = connect_options.clone().pragma("query_only", "ON");+ // A read-only unix-excl handle falls back to POSIX file locks and a+ // file-backed WAL index. Keep macOS query handles physically RW so+ // every connection shares the process-local WAL index; query_only+ // remains the connection-level write barrier.+ #[cfg(not(target_os = "macos"))]+ let options = options.read_only(true);+ options};// Read pool: handles all SELECT queries (search, timeline, API, pipes).@@ -805,7 +821,7 @@ mod shutdown_tests {use std::sync::atomic::{AtomicUsize, Ordering};#[tokio::test]- async fn file_query_pool_remains_read_only_when_query_only_is_disabled() {+ async fn file_query_pool_enforces_its_platform_write_barrier() {let dir = tempfile::tempdir().expect("temp dir");let db_path = dir.path().join("read-only-query-pool.sqlite");let database = DatabaseManager::new(@@ -826,22 +842,41 @@ mod shutdown_tests {.await.expect("read query_only");assert_eq!(query_only, 1, "query connections must enable query_only");-- sqlx::query("PRAGMA query_only = OFF")- .execute(&mut *reader)- .await- .expect("disable connection-level guard for physical-mode test");- let write_error = sqlx::query("INSERT INTO query_pool_probe (value) VALUES (1)")+ let guarded_write = sqlx::query("INSERT INTO query_pool_probe (value) VALUES (0)").execute(&mut *reader).await- .expect_err("mode=ro query connection unexpectedly accepted a write");+ .expect_err("query_only connection unexpectedly accepted a write");assert!(- write_error+ guarded_write.to_string().to_ascii_lowercase().contains("readonly"),- "unexpected query-pool write error: {write_error}"+ "unexpected query_only write error: {guarded_write}");++ sqlx::query("PRAGMA query_only = OFF")+ .execute(&mut *reader)+ .await+ .expect("disable connection-level guard for physical-mode test");+ #[cfg(not(target_os = "macos"))]+ {+ let write_error = sqlx::query("INSERT INTO query_pool_probe (value) VALUES (1)")+ .execute(&mut *reader)+ .await+ .expect_err("mode=ro query connection unexpectedly accepted a write");+ assert!(+ write_error+ .to_string()+ .to_ascii_lowercase()+ .contains("readonly"),+ "unexpected query-pool write error: {write_error}"+ );+ }+ #[cfg(target_os = "macos")]+ sqlx::query("INSERT INTO query_pool_probe (value) VALUES (1)")+ .execute(&mut *reader)+ .await+ .expect("unix-excl query handles are physically RW behind query_only");sqlx::query("PRAGMA query_only = ON").execute(&mut *reader).await@@ -856,7 +891,18 @@ mod shutdown_tests {.fetch_one(&database.pool).await.expect("query pool remains readable");+ #[cfg(not(target_os = "macos"))]assert_eq!(count, 1);+ #[cfg(target_os = "macos")]+ assert_eq!(count, 2);++ #[cfg(target_os = "macos")]+ assert!(+ !db_path+ .with_file_name("read-only-query-pool.sqlite-shm")+ .exists(),+ "unix-excl must keep the live WAL index out of an APFS-backed -shm mapping"+ );database.close().await;}