YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Fix macOS NAS migration and recording durability (#7090)
Ezra Ellette90aa96ed8405
diff --git a/.github/workflows/linux-appimage-smoke.yml b/.github/workflows/linux-appimage-smoke.ymlindex efecb497c..6d867ee35 100644--- a/.github/workflows/linux-appimage-smoke.yml+++ b/.github/workflows/linux-appimage-smoke.yml@@ -439,6 +439,11 @@ jobs:pacman -Q libavif libsharpyuv || truels -l /usr/lib/libavif.so.* /usr/lib/libsharpyuv.so.* || true+ # Containers do not boot systemd, which normally creates this.+ # Exercise normal first-launch identity with an owned container ID.+ dbus-uuidgen > /etc/machine-id+ test -s /etc/machine-id+useradd -m smoke-userchown -R smoke-user:smoke-user /smokeinstall -d -m 700 -o smoke-user -g smoke-user /tmp/screenpipe-runtimediff --git a/.github/workflows/macos-intel-launch-smoke.yml b/.github/workflows/macos-intel-launch-smoke.ymlindex 333908e75..461fe4041 100644--- a/.github/workflows/macos-intel-launch-smoke.yml+++ b/.github/workflows/macos-intel-launch-smoke.yml@@ -176,7 +176,11 @@ jobs:env:NEXT_PUBLIC_SCREENPIPE_E2E: "true"with:- args: "--no-bundle -- --profile debug-dev --features e2e"+ # The default macOS config includes the arm64-only MLX sidecar.+ # Match the Intel release's external binaries without requiring MLX.+ args: >-+ --config '{"bundle":{"externalBin":["bun","ffmpeg","ffprobe"]}}'+ --no-bundle -- --profile debug-dev --features e2eprojectPath: "./apps/screenpipe-app-tauri"tauriScript: bunx tauri -vdiff --git a/COVERAGE.md b/COVERAGE.mdindex 8f84df623..e6ee066f3 100644--- a/COVERAGE.md+++ b/COVERAGE.md@@ -34,6 +34,17 @@ results and `cargo llvm-cov` data on top when judging release confidence.### Core Engine+- Mapped suites: 34+- Mapped Rust files: 359+- Active test blocks: 3524+- Ignored/manual test blocks: 148+- Weighted coverage points: 2910.1++| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score |+| --- | --- | --- | --- | --- | --- | --- | --- |+| windows | 31 | 3381 | 142 | 2845.0 | 21 | 11 | 100% |+| macos | 31 | 3443 | 122 | 2859.0 | 22 | 11 | 100% |+| linux | 27 | 3011 | 114 | 2504.3 | 20 | 11 | 100% |- Mapped suites: 33- Mapped Rust files: 359- Active test blocks: 3523diff --git a/Cargo.lock b/Cargo.lockindex c76886590..a84f896e0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8658,6 +8658,7 @@ dependencies = ["screenpipe-core","screenpipe-db","screenpipe-events",+ "screenpipe-fs","serde","serde_json","sha2 0.10.9",@@ -8765,6 +8766,7 @@ dependencies = ["russh-sftp","rustls-native-certs","screenpipe-core",+ "screenpipe-fs","screenpipe-secrets","screenpipe-vault","serde",@@ -8817,6 +8819,7 @@ dependencies = ["reqwest 0.13.3","screenpipe-cpu-features","screenpipe-events",+ "screenpipe-fs","screenpipe-secrets","screenpipe-sync","screenpipe-vault",@@ -8860,6 +8863,7 @@ dependencies = ["rusqlite","screenpipe-config","screenpipe-core",+ "screenpipe-fs","screenpipe-resource","screenpipe-secrets","screenpipe-semantic",@@ -8927,6 +8931,7 @@ dependencies = ["screenpipe-core","screenpipe-db","screenpipe-events",+ "screenpipe-fs","screenpipe-redact","screenpipe-resource","screenpipe-screen",@@ -8982,6 +8987,14 @@ dependencies = ["tracing",]+[[package]]+name = "screenpipe-fs"+version = "0.4.50"+dependencies = [+ "libc",+ "tempfile",+]+[[package]]name = "screenpipe-gateway"version = "0.4.50"@@ -9058,6 +9071,7 @@ dependencies = ["regex","reqwest 0.13.3","screenpipe-db",+ "screenpipe-fs","screenpipe-resource","screenpipe-rfdetr-mlx","screenpipe-sqlite-coordinator",@@ -9125,6 +9139,7 @@ dependencies = ["screenpipe-connect","screenpipe-core","screenpipe-db",+ "screenpipe-fs","serde","serde_json","strsim 0.10.0",@@ -9183,6 +9198,7 @@ version = "0.4.50"dependencies = ["fs2","libsqlite3-sys",+ "screenpipe-fs","serde","serde_json","sqlx",diff --git a/apps/screenpipe-app-tauri/components/storage-migration-prompt.tsx b/apps/screenpipe-app-tauri/components/storage-migration-prompt.tsxindex df2a954cf..f137293aa 100644--- a/apps/screenpipe-app-tauri/components/storage-migration-prompt.tsx+++ b/apps/screenpipe-app-tauri/components/storage-migration-prompt.tsx@@ -14,7 +14,7 @@ export function StorageMigrationDescription() {return <><span className="mt-3 block">Screenpipe will pause recording and history access while it converts and verifies your history, then restore your recording preference.</span><span className="mt-3 block">Keep the app open. Your computer will stay awake. Progress and elapsed time will be shown; the time needed depends on your database and computer.</span>- <span className="mt-3 block">Space is recovered as each batch is verified. Your existing database becomes the smaller index. If migration is interrupted, Screenpipe must restore storage before recording can resume. You can retry migration afterward.</span>+ <span className="mt-3 block">Your existing database becomes the index for your upgraded history. Space is recovered as each batch is verified where the drive supports it; some network drives keep freed space for reuse. If migration is interrupted, Screenpipe must restore storage before recording can resume. You can retry migration afterward.</span></>;}@@ -105,14 +105,14 @@ export function StorageMigrationPrompt({ activity }: { activity: StorageMigratio<AlertDialogTitle>{success ? "Storage migration complete" : failure ? "Migration needs attention" : status?.pending ? "Finish migrating your history" : "Upgrade your history storage"}</AlertDialogTitle><AlertDialogDescription>{success ? activity.message : <>- {!failure && "Your history is in the older storage format. Upgrade it to reduce database size and keep it searchable. "}+ {!failure && "Your history is in the older storage format. Upgrade it to store history more efficiently and keep it searchable. "}<StorageMigrationDescription /></>}</AlertDialogDescription></AlertDialogHeader>{success && <div className="space-y-2 text-sm text-muted-foreground"><p>Completed in {migrationElapsed(activity.elapsed_seconds)}.</p>- <p>{status?.in_place ? "Your existing database is now the smaller index. Space was recovered during migration." : "Your original database is kept as a recovery copy. You can delete it separately in Settings → Storage after reviewing your history."}</p>+ <p>{status?.in_place ? "Your existing database is now the index for your upgraded history." : "Your original database is kept as a recovery copy. You can delete it separately in Settings → Storage after reviewing your history."}</p>{status?.bytes_saved != null && <p>Space saved: {migrationBytes(status.bytes_saved)}</p>}</div>}{failure && <p className="text-sm text-destructive" role="alert">{failure}</p>}diff --git a/apps/screenpipe-app-tauri/e2e/specs/onboarding-background-ai-tools.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/onboarding-background-ai-tools.spec.tsindex 70423f48e..e35b610d2 100644--- a/apps/screenpipe-app-tauri/e2e/specs/onboarding-background-ai-tools.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/onboarding-background-ai-tools.spec.ts@@ -332,7 +332,7 @@ async function callActivitySummaryThroughMcp(// The contract: a saved `connect-apps` install must not get stuck on the// removed slide. It resumes at the engine, advances to the new optional- // recommended-setup step, and finishes when the user continues. Sample+ // recommended-setup step, and finishes when the user starts. Sample// the body while the window is alive so we can prove the removed slides// never rendered.//@@ -385,13 +385,35 @@ async function callActivitySummaryThroughMcp();await browser.switchToWindow("onboarding");- const continueButton = await $("button=continue");- await continueButton.waitForClickable({ timeout: t(10_000) });- await continueButton.click();+ // This lane proves native MCP setup and saved-step recovery. Opt out of+ // scheduled AI tasks through their real controls, so finishing setup+ // needs neither hosted model credentials nor an external task install.+ const defaults = await $$('[data-testid="onboarding-final-setup"] [role="switch"]');+ expect(defaults.length).toBe(4);+ for (const control of defaults) {+ expect(await control.getAttribute("aria-checked")).toBe("true");+ await control.scrollIntoView();+ await control.click();+ expect(await control.getAttribute("aria-checked")).toBe("false");+ }+ const startButton = await $("button=Start Screenpipe");+ await startButton.scrollIntoView();+ await startButton.waitForClickable({ timeout: t(10_000) });+ await startButton.click();await browser.switchToWindow("home");await browser.waitUntil(async () => {+ // The optional Workflows rollout adds a final choice. Complete its+ // normal Screenpipe path if present; never bypass persisted completion.+ if ((await browser.getWindowHandles()).includes("onboarding")) {+ await browser.switchToWindow("onboarding");+ const findSomething = await $("button*=Find something");+ if (await findSomething.isExisting() && await findSomething.isDisplayed()) {+ await findSomething.click();+ }+ }+ await browser.switchToWindow("home");const status = await invokeOrThrow<{ isCompleted: boolean }>("get_onboarding_status",);diff --git a/apps/screenpipe-app-tauri/knip.json b/apps/screenpipe-app-tauri/knip.jsonindex 16ebe0067..f53c36988 100644--- a/apps/screenpipe-app-tauri/knip.json+++ b/apps/screenpipe-app-tauri/knip.json@@ -11,6 +11,7 @@"e2e/mock-updates/windows-persistence-server.ts","e2e/scripts/verify-spotlight-exclusion.ts","scripts/timeline-parity-export.ts",+ "scripts/eval-workflow-assistant.ts","scripts/eval-workflow-discovery.ts","scripts/eval-workflow-guide.ts","scripts/eval-workflow-pipeline.ts",diff --git a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock b/apps/screenpipe-app-tauri/src-tauri/Cargo.lockindex 8035441cf..df78fcaf4 100644--- a/apps/screenpipe-app-tauri/src-tauri/Cargo.lock+++ b/apps/screenpipe-app-tauri/src-tauri/Cargo.lock@@ -11265,6 +11265,7 @@ dependencies = ["screenpipe-db","screenpipe-engine","screenpipe-events",+ "screenpipe-fs","screenpipe-overlay-win","screenpipe-redact","screenpipe-screen",@@ -11375,6 +11376,7 @@ dependencies = ["screenpipe-core","screenpipe-db","screenpipe-events",+ "screenpipe-fs","serde","serde_json","sha2 0.10.9",@@ -11457,6 +11459,7 @@ dependencies = ["russh-sftp","rustls-native-certs","screenpipe-core",+ "screenpipe-fs","screenpipe-secrets","screenpipe-vault","serde",@@ -11507,6 +11510,7 @@ dependencies = ["reqwest 0.13.4","screenpipe-cpu-features","screenpipe-events",+ "screenpipe-fs","screenpipe-secrets","screenpipe-sync","screenpipe-vault",@@ -11548,6 +11552,7 @@ dependencies = ["regex","rusqlite","screenpipe-config",+ "screenpipe-fs","screenpipe-resource","screenpipe-semantic","screenpipe-sqlite-coordinator",@@ -11608,6 +11613,7 @@ dependencies = ["screenpipe-core","screenpipe-db","screenpipe-events",+ "screenpipe-fs","screenpipe-redact","screenpipe-resource","screenpipe-screen",@@ -11656,6 +11662,13 @@ dependencies = ["tracing",]+[[package]]+name = "screenpipe-fs"+version = "0.4.50"+dependencies = [+ "libc",+]+[[package]]name = "screenpipe-overlay-win"version = "0.4.50"@@ -11703,6 +11716,7 @@ dependencies = ["regex","reqwest 0.13.4","screenpipe-db",+ "screenpipe-fs","screenpipe-resource","screenpipe-rfdetr-mlx","screenpipe-sqlite-coordinator",@@ -11766,6 +11780,7 @@ dependencies = ["screenpipe-connect","screenpipe-core","screenpipe-db",+ "screenpipe-fs","serde","serde_json","tokio",@@ -11812,6 +11827,7 @@ version = "0.4.50"dependencies = ["fs2","libsqlite3-sys",+ "screenpipe-fs","serde","serde_json","sqlx",diff --git a/apps/screenpipe-app-tauri/src-tauri/Cargo.toml b/apps/screenpipe-app-tauri/src-tauri/Cargo.tomlindex c495ee899..8936b3a86 100644--- a/apps/screenpipe-app-tauri/src-tauri/Cargo.toml+++ b/apps/screenpipe-app-tauri/src-tauri/Cargo.toml@@ -29,6 +29,7 @@ tauri-helper = { git = "https://github.com/screenpipe/tauri-helper", rev = "e25fcc = "1"[dependencies]+screenpipe-fs = { path = "../../../crates/screenpipe-fs" }fs2 = "0.4"tauri-helper = { git = "https://github.com/screenpipe/tauri-helper", rev = "e25ff1d2c5026189305a0251a1e06788efe1d2b3" }tauri = { version = "=2.11.2", features = [diff --git a/apps/screenpipe-app-tauri/src-tauri/src/store.rs b/apps/screenpipe-app-tauri/src-tauri/src/store.rsindex 5c90030fc..5cb14019d 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/store.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/store.rs@@ -314,7 +314,7 @@ pub(crate) fn durable_write(path: &Path, bytes: &[u8]) -> std::io::Result<()> {}let mut f = opts.open(&tmp)?;f.write_all(bytes)?;- f.sync_all()?; // contents + metadata to stable storage before the rename+ screenpipe_fs::sync_all(&f)?; // contents + metadata to stable storage before the rename}replace_store_temp(&tmp, path)?;// fsync the directory so the rename itself survives a crash. Best-effort:@@ -323,7 +323,7 @@ pub(crate) fn durable_write(path: &Path, bytes: &[u8]) -> std::io::Result<()> {#[cfg(unix)]if let Some(dir) = path.parent() {if let Ok(d) = std::fs::File::open(dir) {- let _ = d.sync_all();+ let _ = screenpipe_fs::sync_all(&d);}}Ok(())diff --git a/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock b/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lockindex f86fedb03..1dc8ce7f9 100644--- a/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock+++ b/apps/screenpipe-workflows-tauri/src-tauri/Cargo.lock@@ -11361,6 +11361,7 @@ dependencies = ["screenpipe-core","screenpipe-db","screenpipe-events",+ "screenpipe-fs","serde","serde_json","sha2 0.10.9",@@ -11443,6 +11444,7 @@ dependencies = ["russh-sftp","rustls-native-certs","screenpipe-core",+ "screenpipe-fs","screenpipe-secrets","screenpipe-vault","serde",@@ -11493,6 +11495,7 @@ dependencies = ["reqwest 0.13.4","screenpipe-cpu-features","screenpipe-events",+ "screenpipe-fs","screenpipe-secrets","screenpipe-sync","screenpipe-vault",@@ -11534,6 +11537,7 @@ dependencies = ["regex","rusqlite","screenpipe-config",+ "screenpipe-fs","screenpipe-resource","screenpipe-semantic","screenpipe-sqlite-coordinator",@@ -11594,6 +11598,7 @@ dependencies = ["screenpipe-core","screenpipe-db","screenpipe-events",+ "screenpipe-fs","screenpipe-redact","screenpipe-resource","screenpipe-screen",@@ -11642,6 +11647,13 @@ dependencies = ["tracing",]+[[package]]+name = "screenpipe-fs"+version = "0.4.50"+dependencies = [+ "libc",+]+[[package]]name = "screenpipe-overlay-win"version = "0.4.50"@@ -11683,6 +11695,7 @@ dependencies = ["regex","reqwest 0.13.4","screenpipe-db",+ "screenpipe-fs","screenpipe-resource","screenpipe-rfdetr-mlx","screenpipe-sqlite-coordinator",@@ -11746,6 +11759,7 @@ dependencies = ["screenpipe-connect","screenpipe-core","screenpipe-db",+ "screenpipe-fs","serde","serde_json","tokio",@@ -11792,6 +11806,7 @@ version = "0.4.50"dependencies = ["fs2","libsqlite3-sys",+ "screenpipe-fs","serde","serde_json","sqlx",diff --git a/crates/screenpipe-audio/Cargo.toml b/crates/screenpipe-audio/Cargo.tomlindex e22e2e44c..95c9587da 100644--- a/crates/screenpipe-audio/Cargo.toml+++ b/crates/screenpipe-audio/Cargo.toml@@ -17,6 +17,7 @@ edition = { workspace = true }which = { workspace = true }[dependencies]+screenpipe-fs = { path = "../screenpipe-fs" }serde = { workspace = true }serde_json = { workspace = true }oasgen = { workspace = true }diff --git a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs b/crates/screenpipe-audio/src/audio_manager/reconciliation.rsindex 8d1addc21..d75f7ff42 100644--- a/crates/screenpipe-audio/src/audio_manager/reconciliation.rs+++ b/crates/screenpipe-audio/src/audio_manager/reconciliation.rs@@ -166,10 +166,7 @@ fn stage_merged_audio() -> anyhow::Result<PathBuf> {let path = merged_audio_temp_path(primary_path);write_audio_to_file(samples, sample_rate, &path, false)?;- std::fs::OpenOptions::new()- .write(true)- .open(&path)?- .sync_all()?;+ screenpipe_fs::sync_all(&std::fs::OpenOptions::new().write(true).open(&path)?)?;sync_parent(&path)?;Ok(path)}@@ -177,7 +174,7 @@ fn stage_merged_audio(fn sync_parent(path: &Path) -> std::io::Result<()> {#[cfg(unix)]if let Some(parent) = path.parent() {- std::fs::File::open(parent)?.sync_all()?;+ screenpipe_fs::sync_all(&std::fs::File::open(parent)?)?;}Ok(())}@@ -768,7 +765,7 @@ fn write_pending(data_dir: &Path, pending: &PendingTranscription) -> std::io::Re.create_new(true).open(&tmp_path)?;file.write_all(json.as_bytes())?;- file.sync_all()?;+ screenpipe_fs::sync_all(&file)?;std::fs::rename(&tmp_path, &path)?;sync_parent(&path)?;debug!(diff --git a/crates/screenpipe-connect/Cargo.toml b/crates/screenpipe-connect/Cargo.tomlindex 9d2673fad..35b3420f8 100644--- a/crates/screenpipe-connect/Cargo.toml+++ b/crates/screenpipe-connect/Cargo.toml@@ -13,6 +13,7 @@ edition = { workspace = true }ignored = ["libsqlite3-sys"][dependencies]+screenpipe-fs = { path = "../screenpipe-fs" }specta = { workspace = true, optional = true }image = { workspace = true }reqwest = { workspace = true }diff --git a/crates/screenpipe-connect/src/mcp_servers.rs b/crates/screenpipe-connect/src/mcp_servers.rsindex 5c6a340ba..ea3429bfa 100644--- a/crates/screenpipe-connect/src/mcp_servers.rs+++ b/crates/screenpipe-connect/src/mcp_servers.rs@@ -347,7 +347,7 @@ async fn save_file(screenpipe_dir: &Path, file: &McpServersFile) -> Result<()> {use std::io::Write as _;let mut f = std::fs::File::create(&tmp2)?;f.write_all(json.as_bytes())?;- f.sync_all()?;+ screenpipe_fs::sync_all(&f)?;}std::fs::rename(&tmp2, &path2)?;Ok(())diff --git a/crates/screenpipe-connect/src/remote_sync.rs b/crates/screenpipe-connect/src/remote_sync.rsindex 19c9694bd..113435df6 100644--- a/crates/screenpipe-connect/src/remote_sync.rs+++ b/crates/screenpipe-connect/src/remote_sync.rs@@ -646,7 +646,11 @@ async fn snapshot_db(live_db: &Path, dest: &Path) -> Result<()> {// would do real POSIX locking against handles that no longer lock at all.#[cfg(target_os = "macos")]{- opts = opts.vfs("unix-excl");+ // The network-volume recorder uses SQLite's filesystem-aware locks+ // and rollback journal; preserve that same domain for snapshots.+ if !screenpipe_fs::is_network_volume(live_db.parent().unwrap_or(live_db))? {+ opts = opts.vfs("unix-excl");+ }}let pool = SqlitePoolOptions::new()diff --git a/crates/screenpipe-core/Cargo.toml b/crates/screenpipe-core/Cargo.tomlindex ccb32cbb0..f2fe8731b 100644--- a/crates/screenpipe-core/Cargo.toml+++ b/crates/screenpipe-core/Cargo.toml@@ -12,6 +12,7 @@ license-file = { workspace = true }edition = { workspace = true }[dependencies]+screenpipe-fs = { path = "../screenpipe-fs" }screenpipe-cpu-features = { path = "../screenpipe-cpu-features" }screenpipe-events = { path = "../screenpipe-events" }futures = { workspace = true }diff --git a/crates/screenpipe-core/src/starter_skills.rs b/crates/screenpipe-core/src/starter_skills.rsindex 0ccf58835..9d9973bcc 100644--- a/crates/screenpipe-core/src/starter_skills.rs+++ b/crates/screenpipe-core/src/starter_skills.rs@@ -109,7 +109,7 @@ pub fn install_one(root: &Path, name: &str, content: &str) -> io::Result<PathBuf.open(&staged)?;use io::Write;file.write_all(content.as_bytes())?;- file.sync_all()?;+ screenpipe_fs::sync_all(&file)?;drop(file);fs::rename(&staged, &path)?;fs::write(&marker, content)?;@@ -120,7 +120,7 @@ pub fn install_one(root: &Path, name: &str, content: &str) -> io::Result<PathBuf.create_new(true).open(&path)?;file.write_all(content.as_bytes())?;- file.sync_all()?;+ screenpipe_fs::sync_all(&file)?;fs::write(&marker, content)?;}Ok(path)@@ -161,7 +161,7 @@ pub fn install_store(root: &Path) -> io::Result<Vec<PathBuf>> {.create_new(true).open(&staged)?;file.write_all(seen.as_bytes())?;- file.sync_all()?;+ screenpipe_fs::sync_all(&file)?;drop(file);fs::rename(staged, index)?;Ok(installed)diff --git a/crates/screenpipe-db/Cargo.toml b/crates/screenpipe-db/Cargo.tomlindex 3f91e3429..350f94dba 100644--- a/crates/screenpipe-db/Cargo.toml+++ b/crates/screenpipe-db/Cargo.toml@@ -15,6 +15,7 @@ edition.workspace = truestorage-fault-injection = [][dependencies]+screenpipe-fs = { path = "../screenpipe-fs" }uuid = { workspace = true }chrono = { workspace = true }image = { workspace = true }diff --git a/crates/screenpipe-db/src/db/setup.rs b/crates/screenpipe-db/src/db/setup.rsindex a953a48ed..5d1e0d8f4 100644--- a/crates/screenpipe-db/src/db/setup.rs+++ b/crates/screenpipe-db/src/db/setup.rs@@ -247,6 +247,11 @@ impl DatabaseManager {.busy_timeout(Duration::from_secs(5)).pragma("cache_size", format!("-{}", config.cache_size_kb)).pragma("mmap_size", config.mmap_size.to_string());+ #[cfg(target_os = "macos")]+ let network_volume = !is_in_memory+ && screenpipe_fs::is_network_volume(database_file.parent().unwrap_or(database_file))?;+ #[cfg(not(target_os = "macos"))]+ let network_volume = false;// macOS delivers an uncatchable SIGBUS when APFS invalidates a page in// SQLite's file-backed WAL index after the `-shm` file was shortened.// `unix-excl` does NOT move that index into process memory: in the@@ -265,7 +270,7 @@ impl DatabaseManager {// was wrong, and `db.sqlite-shm` existing on disk is normal, not// evidence that some other opener touched the database.#[cfg(target_os = "macos")]- if !is_in_memory {+ if !is_in_memory && !network_volume {connect_options = connect_options.vfs("unix-excl");info!("macOS capture database using unix-excl VFS with a process-exclusive database lock"@@ -274,7 +279,13 @@ impl DatabaseManager {for (pragma, value) in screenpipe_config::WAL_SAFETY_PRAGMAS {connect_options = connect_options.pragma(pragma, value);}- if storage.is_some() {+ // macOS SMB rejects POSIX byte locks, and WAL needs shared memory that+ // network filesystems cannot coordinate across hosts. Use SQLite's+ // filesystem-aware default VFS with rollback journaling instead. The+ // existing manager lease and single writer still own this database.+ let journal_mode = if network_volume { "DELETE" } else { "WAL" };+ connect_options = connect_options.pragma("journal_mode", journal_mode);+ if storage.is_some() || network_volume {connect_options = connect_options.pragma("synchronous", "FULL");}@@ -290,14 +301,24 @@ impl DatabaseManager {.acquire_owned().await.map_err(|_| SqlxError::PoolClosed)?;- let mut conn = connect_options+ // A legacy WAL file on SMB needs a private WAL index while the+ // sole bootstrap connection checkpoints it into rollback mode.+ // Close that exclusive owner before admitting the normal pools.+ let bootstrap_options = if network_volume {+ connect_options.clone().pragma("locking_mode", "EXCLUSIVE")+ } else {+ connect_options.clone()+ };+ let mut conn = bootstrap_options.connect().await.map_err(|error| quarantine_startup_error(database_file, error))?;- sqlx::query("PRAGMA journal_mode=WAL")- .execute(&mut conn)- .await- .map_err(|error| quarantine_startup_error(database_file, error))?;+ sqlx::query(sqlx::AssertSqlSafe(format!(+ "PRAGMA journal_mode={journal_mode}"+ )))+ .execute(&mut conn)+ .await+ .map_err(|error| quarantine_startup_error(database_file, error))?;conn.close().await.map_err(|error| quarantine_startup_error(database_file, error))?;diff --git a/crates/screenpipe-db/src/db/source_identity.rs b/crates/screenpipe-db/src/db/source_identity.rsindex d577bd08c..ab606352b 100644--- a/crates/screenpipe-db/src/db/source_identity.rs+++ b/crates/screenpipe-db/src/db/source_identity.rs@@ -68,10 +68,10 @@ impl DatabaseManager {let temporary = journal.with_extension("tmp");let mut file = std::fs::File::create(&temporary)?;file.write_all(&serde_json::to_vec(&saved).expect("serializable journal"))?;- file.sync_all()?;+ screenpipe_fs::sync_all(&file)?;std::fs::rename(&temporary, journal)?;#[cfg(unix)]- if let Some(parent) = journal.parent() { std::fs::File::open(parent)?.sync_all()?; }+ if let Some(parent) = journal.parent() { screenpipe_fs::sync_all(&std::fs::File::open(parent)?)?; }saved}Err(error) => return Err(error.into()),diff --git a/crates/screenpipe-db/src/storage/bulk/codec.rs b/crates/screenpipe-db/src/storage/bulk/codec.rsindex 460f6426e..b1dc02433 100644--- a/crates/screenpipe-db/src/storage/bulk/codec.rs+++ b/crates/screenpipe-db/src/storage/bulk/codec.rs@@ -141,7 +141,7 @@ pub(super) fn write(path: &Path, table: &Table, rows: &[Record]) -> Result<Strin}group.close().map_err(storage_error)?;}- writer.into_inner().map_err(storage_error)?.sync_all()?;+ screenpipe_fs::sync_all(&writer.into_inner().map_err(storage_error)?)?;checksum(path)}diff --git a/crates/screenpipe-db/src/storage/codec.rs b/crates/screenpipe-db/src/storage/codec.rsindex ced7ac391..fe6f6564b 100644--- a/crates/screenpipe-db/src/storage/codec.rs+++ b/crates/screenpipe-db/src/storage/codec.rs@@ -95,7 +95,7 @@ pub fn write(}group.close().map_err(storage_error)?;}- writer.into_inner().map_err(storage_error)?.sync_all()?;+ screenpipe_fs::sync_all(&writer.into_inner().map_err(storage_error)?)?;checksum(path)}diff --git a/crates/screenpipe-db/src/storage/in_place.rs b/crates/screenpipe-db/src/storage/in_place.rsindex 8335886f2..d67cc69e3 100644--- a/crates/screenpipe-db/src/storage/in_place.rs+++ b/crates/screenpipe-db/src/storage/in_place.rs@@ -85,11 +85,21 @@ async fn run(crate::db::register_sqlite_extensions()?;// Do not close this fd while SQLite holds its process-wide Unix locks.let file = OpenOptions::new().read(true).write(true).open(&index)?;+ #[cfg(target_os = "macos")]+ let network_volume = screenpipe_fs::is_network_volume(&storage.root)?;+ #[cfg(not(target_os = "macos"))]+ let network_volume = false;let options = sqlx::sqlite::SqliteConnectOptions::new().filename(&index).create_if_missing(false).foreign_keys(false)- .pragma("journal_mode", "WAL")+ // Keep the same network-compatible journal through verification and+ // restart. A private EXCLUSIVE WAL cannot be reopened by the normal+ // network VFS, which has no shared-memory methods.+ .pragma(+ "journal_mode",+ if network_volume { "DELETE" } else { "WAL" },+ ).pragma("locking_mode", "EXCLUSIVE").pragma("synchronous", "FULL").pragma("temp_store", "FILE")@@ -109,7 +119,13 @@ async fn run(pool.clone(),screenpipe_sqlite_coordinator::sqlite_write_lock(&index),);- let mut reclamation = super::reclaim::Reclaimer::default();+ // Re-probe on each explicit attempt: a resumed migration can be on a+ // filesystem that cannot punch holes. Its batch reserve still applies.+ let mut reclamation = if archive {+ super::reclaim::Reclaimer::new(&storage.root)?+ } else {+ super::reclaim::Reclaimer::default()+ };let mut last_report = None;let result = async {if archive { reserve(&storage)?; }diff --git a/crates/screenpipe-db/src/storage/mod.rs b/crates/screenpipe-db/src/storage/mod.rsindex b3648256e..f25c650e4 100644--- a/crates/screenpipe-db/src/storage/mod.rs+++ b/crates/screenpipe-db/src/storage/mod.rs@@ -354,13 +354,13 @@ pub(crate) fn sync_file(path: &Path) -> Result<(), sqlx::Error> {// completed file was written by SQLite or copied through another handle.#[cfg(windows)]options.write(true);- options.open(path)?.sync_all()?;+ screenpipe_fs::sync_all(&options.open(path)?)?;Ok(())}pub(crate) fn sync_directory(path: &Path) -> Result<(), sqlx::Error> {#[cfg(unix)]- std::fs::File::open(path)?.sync_all()?;+ screenpipe_fs::sync_all(&std::fs::File::open(path)?)?;#[cfg(windows)]{use std::os::windows::fs::OpenOptionsExt;@@ -382,7 +382,7 @@ pub(crate) fn durable_json<T: Serialize>(path: &Path, value: &T) -> Result<(), slet mut temporary = tempfile::NamedTempFile::new_in(parent)?;serde_json::to_writer(&mut temporary, value).map_err(storage_error)?;temporary.write_all(b"\n")?;- temporary.as_file().sync_all()?;+ screenpipe_fs::sync_all(temporary.as_file())?;temporary.persist(path).map_err(|e| sqlx::Error::Io(e.error))?;diff --git a/crates/screenpipe-db/src/storage/reclaim.rs b/crates/screenpipe-db/src/storage/reclaim.rsindex a756a1655..51a86dbe9 100644--- a/crates/screenpipe-db/src/storage/reclaim.rs+++ b/crates/screenpipe-db/src/storage/reclaim.rs@@ -50,22 +50,31 @@ pub(super) fn footprint(root: &Path) -> Result<u64, sqlx::Error> {/// Probe an owned scratch file on the exact destination volume, before touching/// the database. A successful syscall alone does not prove blocks were freed.-pub(super) fn probe(root: &Path) -> Result<(), sqlx::Error> {+pub(super) fn probe(root: &Path) -> Result<bool, sqlx::Error> {+ probe_with(root, punch)+}++fn probe_with(+ root: &Path,+ punch: impl FnOnce(&File, u64, u64) -> Result<(), sqlx::Error>,+) -> Result<bool, sqlx::Error> {let mut file = tempfile::NamedTempFile::new_in(root)?;let bytes = vec![0x5a; 1024 * 1024];file.write_all(&bytes)?;- file.as_file().sync_all()?;+ screenpipe_fs::sync_all(file.as_file())?;let before = allocated(file.path())?;- punch(file.as_file(), 0, bytes.len() as u64).map_err(|e| {- storage_error(format!(- "filesystem reclamation unavailable; migration has not started: {e}"- ))- })?;- file.as_file().sync_all()?;+ match punch(file.as_file(), 0, bytes.len() as u64) {+ Ok(()) => {}+ Err(sqlx::Error::Io(error)) if unsupported(&error) => {+ tracing::info!("filesystem does not support sparse reclamation; keeping reusable SQLite free pages and enforcing migration disk reserve");+ return Ok(false);+ }+ Err(error) => return Err(error),+ }+ screenpipe_fs::sync_all(file.as_file())?;if allocated(file.path())? >= before {- return Err(storage_error(- "this filesystem does not release sparse file blocks; migration has not started",- ));+ tracing::info!("filesystem does not report sparse space savings; keeping reusable SQLite free pages and enforcing migration disk reserve");+ return Ok(false);}file.seek(SeekFrom::Start(0))?;let mut actual = vec![1; bytes.len()];@@ -73,7 +82,20 @@ pub(super) fn probe(root: &Path) -> Result<(), sqlx::Error> {if actual.iter().any(|b| *b != 0) {return Err(storage_error("filesystem reclamation probe failed"));}- Ok(())+ Ok(true)+}++fn unsupported(error: &std::io::Error) -> bool {+ if error.kind() == std::io::ErrorKind::Unsupported {+ return true;+ }+ // Darwin ENOTSUP is Uncategorized in Rust, unlike Linux EOPNOTSUPP.+ #[cfg(unix)]+ return error.raw_os_error().is_some_and(|code| {+ code == libc::ENOTSUP || code == libc::EOPNOTSUPP || code == libc::ENOSYS+ });+ #[cfg(windows)]+ return matches!(error.raw_os_error(), Some(1 | 50)); // INVALID_FUNCTION / NOT_SUPPORTED}#[cfg(target_os = "macos")]@@ -262,7 +284,7 @@ pub(super) async fn free_leaves(}}super::diagnostics::stage("syncing_reclaimed_database");- file.sync_all()?;+ screenpipe_fs::sync_all(file)?;Ok(())}@@ -275,9 +297,17 @@ pub(super) async fn free_leaves(#[derive(Default)]pub(super) struct Reclaimer {next_free_pages: i64,+ disabled: bool,}impl Reclaimer {+ pub(super) fn new(root: &Path) -> Result<Self, sqlx::Error> {+ Ok(Self {+ disabled: !probe(root)?,+ ..Self::default()+ })+ }+pub(super) async fn run(&mut self,conn: &mut SqliteConnection,@@ -289,6 +319,9 @@ impl Reclaimer {// Check space after flushing: checkpointing can reallocate holes that// SQLite reused, even when the free-page total did not change.super::schema::construction_checkpoint(conn).await?;+ if self.disabled {+ return Ok(false);+ }let free: i64 = sqlx::query_scalar("PRAGMA freelist_count").fetch_one(&mut *conn).await?;@@ -317,6 +350,28 @@ mod tests {use super::*;use sqlx::Connection;+ #[test]+ fn unsupported_reclamation_is_optional_but_io_errors_are_not() {+ let root = tempfile::tempdir().unwrap();+ assert!(!probe_with(root.path(), |_, _, _| {+ Err(std::io::Error::from(std::io::ErrorKind::Unsupported).into())+ })+ .unwrap());+ assert!(probe_with(root.path(), |_, _, _| {+ Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied).into())+ })+ .is_err());+ assert!(!probe_with(root.path(), |_, _, _| Ok(())).unwrap());+ #[cfg(unix)]+ {+ assert!(!probe_with(root.path(), |_, _, _| {+ Err(std::io::Error::from_raw_os_error(libc::ENOTSUP).into())+ })+ .unwrap());+ assert!(!unsupported(&std::io::Error::from_raw_os_error(libc::EIO)));+ }+ }+async fn reclamation_workload(batches: i64,payload_bytes: i64,diff --git a/crates/screenpipe-db/tests/in_place_migration.rs b/crates/screenpipe-db/tests/in_place_migration.rsindex f2888a940..09ef15df5 100644--- a/crates/screenpipe-db/tests/in_place_migration.rs+++ b/crates/screenpipe-db/tests/in_place_migration.rs@@ -714,40 +714,200 @@ async fn sparse_privacy_backlog_completes_migration_and_preserves_recording() {#[tokio::test]#[ignore = "requires a marked disposable filesystem without hole punching"]-async fn unsupported_volume_fails_before_conversion() {+#[cfg(target_os = "macos")]+async fn network_wal_verification_and_reopen_preserve_committed_rows() {+ use sqlx::Connection;+ const CHILD: &str = "SCREENPIPE_NAS_WAL_CHILD";+ if let Ok(path) = std::env::var(CHILD) {+ // Reproduce an old owner that exited with committed WAL on the share.+ let mut conn = sqlx::SqliteConnection::connect_with(+ &sqlx::sqlite::SqliteConnectOptions::new()+ .filename(path)+ .create_if_missing(false)+ .pragma("locking_mode", "EXCLUSIVE")+ .pragma("journal_mode", "WAL")+ .pragma("synchronous", "FULL")+ .pragma("wal_autocheckpoint", "0"),+ )+ .await+ .unwrap();+ sqlx::raw_sql("CREATE TABLE nas_wal(value TEXT); INSERT INTO nas_wal VALUES('acknowledged before crash')")+ .execute(&mut conn).await.unwrap();+ std::process::exit(86);+ }+ let volume = std::path::PathBuf::from(std::env::var("SCREENPIPE_UNSUPPORTED_VOLUME").unwrap());+ assert!(volume.join(".screenpipe-disposable-volume").is_file());+ let root = tempfile::tempdir_in(volume).unwrap();+ let path = root.path().join("db.sqlite");+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.close().await;+ let crashed = std::process::Command::new(std::env::current_exe().unwrap())+ .args([+ "--ignored",+ "--exact",+ "network_wal_verification_and_reopen_preserve_committed_rows",+ ])+ .env(CHILD, &path)+ .output()+ .unwrap();+ assert_eq!(+ crashed.status.code(),+ Some(86),+ "{}",+ String::from_utf8_lossy(&crashed.stderr)+ );+ let wal = root.path().join("db.sqlite-wal");+ let before = std::fs::read(&path).unwrap();+ let before_wal = std::fs::read(&wal).unwrap();+ assert!(!before_wal.is_empty());+ screenpipe_sqlite_coordinator::inspect_database_health(&path)+ .await+ .unwrap();+ assert_eq!(+ std::fs::read(&path).unwrap(),+ before,+ "verification must not rewrite the source"+ );+ assert_eq!(+ std::fs::read(&wal).unwrap(),+ before_wal,+ "verification must not checkpoint or remove WAL"+ );+ screenpipe_sqlite_coordinator::persist_sqlite_verification_pending(+ &path,+ Some(3850),+ "previous NAS locking failure",+ )+ .unwrap();+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, String>("SELECT value FROM nas_wal")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ "acknowledged before crash"+ );+ assert_eq!(+ sqlx::query_scalar::<_, String>("PRAGMA journal_mode")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ "delete"+ );+ assert!(!screenpipe_sqlite_coordinator::sqlite_verification_pending_exists(&path));+ db.execute_raw_sql_write("INSERT INTO nas_wal VALUES('recorded after recovery')")+ .await+ .unwrap();+ db.close().await;+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ assert_eq!(+ sqlx::query_scalar::<_, i64>("SELECT count(*) FROM nas_wal")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 2+ );+ db.close().await;+ println!("NAS WAL: verification preserved database and committed WAL bytes; pending incident recovered; acknowledged and new rows survive restart in rollback mode");+}++#[tokio::test]+#[ignore = "requires a marked disposable filesystem without hole punching"]+async fn non_sparse_volume_migrates_and_keeps_recording_after_restart() {+ let _ = tracing_subscriber::fmt()+ .with_env_filter("screenpipe_db::storage=info")+ .with_ansi(false)+ .try_init();let volume = std::path::PathBuf::from(std::env::var("SCREENPIPE_UNSUPPORTED_VOLUME").unwrap());assert!(volume.join(".screenpipe-disposable-volume").is_file());assert!(fs2::total_space(&volume).unwrap() <= 512 * 1024 * 1024);let root = tempfile::tempdir_in(volume).unwrap();fixture(root.path()).await;+ let path = root.path().join("db.sqlite");let mut options = MigrationOptions::default();options.budget.disk_reserve_bytes = 0;options.budget.file_bytes = 1024 * 1024;options.budget.record_bytes = 1024 * 1024;- let error = migrate(root.path(), Default::default(), options)+ let report = migrate(root.path(), Default::default(), options).await- .unwrap_err();- assert!(- error- .to_string()- .contains("filesystem reclamation unavailable"),- "{error}"- );- assert!(root.path().join("db.sqlite").is_file());+ .unwrap();+ assert_eq!(report.frames, 48);+ assert!(root+ .path()+ .join("storage-migration-complete.json")+ .is_file());assert!(!root.path().join("storage-migration.json").exists());- let db = DatabaseManager::new(- root.path().join("db.sqlite").to_str().unwrap(),- Default::default(),- )- .await- .unwrap();+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.verify_storage().await.unwrap();+ db.execute_raw_sql_write("INSERT INTO frames(id,timestamp,full_text,accessibility_tree_json) VALUES(49,'2026-09-18T12:00:00Z','nasafter recording','new detail'); INSERT INTO elements(id,frame_id,source,role,text) VALUES(49,49,'accessibility','AXText','nasafter element'); INSERT INTO audio_chunks(id,file_path) VALUES(1,'nas-audio.mp4'); INSERT INTO audio_transcriptions(id,audio_chunk_id,offset_index,timestamp,transcription,device) VALUES(1,1,0,'2026-09-18T12:00:00Z','nasafter transcript','test');").await.unwrap();+ while db.seal_payloads().await.unwrap() != 0 {}+ db.close().await;+ let db = DatabaseManager::new(path.to_str().unwrap(), Default::default())+ .await+ .unwrap();+ db.verify_storage().await.unwrap();assert_eq!(sqlx::query_scalar::<_, i64>("SELECT count(*) FROM frames").fetch_one(&db.pool).await.unwrap(),- 48+ 49+ );+ let payloads = db+ .frame_payloads(&[1, 48, 49], Projection::All)+ .await+ .unwrap();+ for id in [1, 48] {+ assert_eq!(+ payloads[&id].accessibility_tree_json.as_deref(),+ Some("capture detail ".repeat(32768).as_str())+ );+ }+ assert_eq!(+ payloads[&49].full_text.as_deref(),+ Some("nasafter recording")+ );+ assert_eq!(+ sqlx::query_scalar::<_, String>("SELECT text FROM elements WHERE id=49")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ "nasafter element"+ );+ assert_eq!(+ sqlx::query_scalar::<_, String>(+ "SELECT transcription FROM audio_transcriptions WHERE id=1"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ "nasafter transcript"+ );+ assert_eq!(+ sqlx::query_scalar::<_, i64>(+ "SELECT count(*) FROM frames_fts WHERE frames_fts MATCH 'nasafter'"+ )+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ 1+ );+ assert_eq!(+ sqlx::query_scalar::<_, String>("PRAGMA integrity_check")+ .fetch_one(&db.pool)+ .await+ .unwrap(),+ "ok");+ println!("non-sparse migration: {} historical frames verified; receipt present; journal removed; new frame, element and transcript sealed, searchable and intact after restart; source_bytes={} index_bytes={} parquet_bytes={}", report.frames, report.source_bytes, report.index_bytes, report.payload_bytes);db.close().await;}diff --git a/crates/screenpipe-engine/Cargo.toml b/crates/screenpipe-engine/Cargo.tomlindex 95dbd4dcf..28d5bd228 100644--- a/crates/screenpipe-engine/Cargo.toml+++ b/crates/screenpipe-engine/Cargo.toml@@ -14,6 +14,7 @@ edition = { workspace = true }# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[dependencies]+screenpipe-fs = { path = "../screenpipe-fs" }serde = { workspace = true }serde_json = { workspace = true }serde_yaml = { workspace = true }diff --git a/crates/screenpipe-engine/src/atomic_file.rs b/crates/screenpipe-engine/src/atomic_file.rsindex 6f2636b84..7599b6116 100644--- a/crates/screenpipe-engine/src/atomic_file.rs+++ b/crates/screenpipe-engine/src/atomic_file.rs@@ -57,7 +57,7 @@ pub(crate) fn replace(path: &Path, bytes: &[u8]) -> io::Result<()> {.tempfile_in(parent)?;temporary.write_all(bytes)?;temporary.flush()?;- temporary.as_file().sync_all()?;+ screenpipe_fs::sync_all(temporary.as_file())?;persist_replace(temporary, path)?;@@ -68,7 +68,7 @@ pub(crate) fn replace(path: &Path, bytes: &[u8]) -> io::Result<()> {// Some external and network filesystems reject directory fsync even// after the file replacement succeeded. Keep the state write usable// there while taking the stronger durability guarantee when offered.- let _ = directory.sync_all();+ let _ = screenpipe_fs::sync_all(&directory);}Ok(())diff --git a/crates/screenpipe-engine/src/auth_key.rs b/crates/screenpipe-engine/src/auth_key.rsindex ec6b2fb8f..41bf0dfed 100644--- a/crates/screenpipe-engine/src/auth_key.rs+++ b/crates/screenpipe-engine/src/auth_key.rs@@ -189,7 +189,7 @@ fn get_or_create_api_auth_recovery(data_dir: &Path, candidate: &str) -> Result<Sif let Err(error) = file.write_all(&payload).and_then(|_| file.write_all(b"\n"))- .and_then(|_| file.sync_all())+ .and_then(|_| screenpipe_fs::sync_all(&file)){drop(file);let _ = std::fs::remove_file(&path);diff --git a/crates/screenpipe-engine/src/cli/db.rs b/crates/screenpipe-engine/src/cli/db.rsindex 18027a073..4df651a5f 100644--- a/crates/screenpipe-engine/src/cli/db.rs+++ b/crates/screenpipe-engine/src/cli/db.rs@@ -330,7 +330,15 @@ fn pid_alive(pid: u32) -> bool {// This lock also protects desktop startup on Windows, where an external// `kill` executable is not available. Check the owner on every platform.let mut system = System::new();- system.refresh_process(Pid::from_u32(pid))+ let pid = Pid::from_u32(pid);+ let found = system.refresh_process(pid);+ // sysinfo 0.29's first Windows refresh only opens a process handle. An+ // exited process can still be opened while another handle (e.g. Child) is+ // alive. Refresh the existing entry to check GetExitCodeProcess as well.+ #[cfg(windows)]+ return found && system.refresh_process(pid);+ #[cfg(not(windows))]+ found}// ── runtime checks ─────────────────────────────────────────────────────@@ -2229,6 +2237,8 @@ mod recovery_tests {dir.path().join(LOCK_FILE).exists(),"force quit leaves the lock file");+ // Retain Child's Windows handle: a terminated process object may still+ // exist, but it must not keep the startup lock owned by a live process.assert!(!pid_alive(child.id()));let _guard = prepare_database_startup(dir.path()).await.unwrap();diff --git a/crates/screenpipe-fs/Cargo.toml b/crates/screenpipe-fs/Cargo.tomlnew file mode 100644index 000000000..d9d6cc974--- /dev/null+++ b/crates/screenpipe-fs/Cargo.toml@@ -0,0 +1,12 @@+[package]+name = "screenpipe-fs"+version.workspace = true+edition.workspace = true+license-file.workspace = true+description = "Durable file writes on local and network filesystems"++[target.'cfg(target_os = "macos")'.dependencies]+libc = { workspace = true }++[dev-dependencies]+tempfile = { workspace = true }diff --git a/crates/screenpipe-fs/src/lib.rs b/crates/screenpipe-fs/src/lib.rsnew file mode 100644index 000000000..90553ea38--- /dev/null+++ b/crates/screenpipe-fs/src/lib.rs@@ -0,0 +1,101 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++//! File durability shared by capture, SQLite lifecycle, and Parquet publication.+//! No filesystem detection, allocation, or additional syscall on local success.++use std::{fs::File, io};++/// Inspect an existing path before opening SQLite. Network volumes need the+/// macOS VFS's filesystem-specific locks, not unix-excl's POSIX byte locks.+#[cfg(target_os = "macos")]+pub fn is_network_volume(path: &std::path::Path) -> io::Result<bool> {+ use std::os::unix::ffi::OsStrExt;+ let path = std::ffi::CString::new(path.as_os_str().as_bytes())?;+ let mut stat = std::mem::MaybeUninit::<libc::statfs>::uninit();+ // SAFETY: the pathname is NUL-terminated and stat points to writable storage.+ if unsafe { libc::statfs(path.as_ptr(), stat.as_mut_ptr()) } != 0 {+ return Err(io::Error::last_os_error());+ }+ // SAFETY: statfs initialized the structure on success.+ Ok(unsafe { stat.assume_init() }.f_flags & libc::MNT_LOCAL as u32 == 0)+}++/// Flush contents and metadata before publishing a file. Rust uses F_FULLFSYNC+/// on macOS; SMB supports fsync (server FLUSH) but rejects that device-cache+/// operation. Fall back only for ENOTSUP, never for disk, permission, or network+/// failures. A successful server flush still depends on the server's durability+/// contract, just as SQLite's own full_fsync fallback does.+pub fn sync_all(file: &File) -> io::Result<()> {+ let result = file.sync_all();+ #[cfg(target_os = "macos")]+ return supported_sync(result, || {+ use std::os::fd::AsRawFd;+ loop {+ // SAFETY: the borrowed File keeps this descriptor open.+ if unsafe { libc::fsync(file.as_raw_fd()) } == 0 {+ return Ok(());+ }+ let error = io::Error::last_os_error();+ if error.kind() != io::ErrorKind::Interrupted {+ return Err(error);+ }+ }+ });+ #[cfg(not(target_os = "macos"))]+ result+}++#[cfg(target_os = "macos")]+fn supported_sync(+ full_sync: io::Result<()>,+ fsync: impl FnOnce() -> io::Result<()>,+) -> io::Result<()> {+ match full_sync {+ Err(error) if error.raw_os_error() == Some(libc::ENOTSUP) => fsync(),+ result => result,+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn flushes_written_file() {+ use std::io::Write;+ let mut file = tempfile::tempfile().unwrap();+ file.write_all(b"durable recording").unwrap();+ sync_all(&file).unwrap();+ }++ #[cfg(target_os = "macos")]+ #[test]+ fn unsupported_full_sync_requires_successful_fsync() {+ supported_sync(Err(io::Error::from_raw_os_error(libc::ENOTSUP)), || Ok(())).unwrap();+ let error = supported_sync(Err(io::Error::from_raw_os_error(libc::ENOTSUP)), || {+ Err(io::Error::from_raw_os_error(libc::EIO))+ })+ .unwrap_err();+ assert_eq!(error.raw_os_error(), Some(libc::EIO));+ }++ #[cfg(target_os = "macos")]+ #[test]+ fn successful_full_sync_and_real_errors_never_fall_back() {+ supported_sync(Ok(()), || panic!("unnecessary fallback")).unwrap();+ for code in [+ libc::EIO,+ libc::ENOSPC,+ libc::EACCES,+ libc::EBADF,+ libc::ETIMEDOUT,+ ] {+ let error = supported_sync(Err(io::Error::from_raw_os_error(code)), || {+ panic!("must preserve real sync failure")+ })+ .unwrap_err();+ assert_eq!(error.raw_os_error(), Some(code));+ }+ }+}diff --git a/crates/screenpipe-redact/Cargo.toml b/crates/screenpipe-redact/Cargo.tomlindex 6a834917d..641e2de10 100644--- a/crates/screenpipe-redact/Cargo.toml+++ b/crates/screenpipe-redact/Cargo.toml@@ -12,6 +12,7 @@ license-file.workspace = trueedition.workspace = true[dependencies]+screenpipe-fs = { path = "../screenpipe-fs" }# Re-exports & shared depsaho-corasick = "1"anyhow = { workspace = true }diff --git a/crates/screenpipe-redact/src/image/frame_redactor.rs b/crates/screenpipe-redact/src/image/frame_redactor.rsindex 432ce2f73..0eb959562 100644--- a/crates/screenpipe-redact/src/image/frame_redactor.rs+++ b/crates/screenpipe-redact/src/image/frame_redactor.rs@@ -126,10 +126,10 @@ pub fn redact_frame(tmp_writer.flush().map_err(|e| RedactError::Runtime(format!("flush {}: {e}", tmp_path.display())))?;- tmp_writer+ let file = tmp_writer.into_inner()- .map_err(|e| RedactError::Runtime(format!("close {}: {e}", tmp_path.display())))?- .sync_all()+ .map_err(|e| RedactError::Runtime(format!("close {}: {e}", tmp_path.display())))?;+ screenpipe_fs::sync_all(&file).map_err(|e| RedactError::Runtime(format!("fsync {}: {e}", tmp_path.display())))?;std::fs::rename(&tmp_path, &output_path).map_err(|e| {// On rename failure, try to clean up the tempfile — best effort.diff --git a/crates/screenpipe-redact/src/pseudonym.rs b/crates/screenpipe-redact/src/pseudonym.rsindex e0cffcd2c..f873d9979 100644--- a/crates/screenpipe-redact/src/pseudonym.rs+++ b/crates/screenpipe-redact/src/pseudonym.rs@@ -225,7 +225,7 @@ fn create_new_key_file(path: &Path, key: &[u8; KEY_LEN]) -> std::io::Result<()>}let mut f = opts.open(path)?;f.write_all(key)?;- f.sync_all()?;+ screenpipe_fs::sync_all(&f)?;Ok(())}diff --git a/crates/screenpipe-screen/Cargo.toml b/crates/screenpipe-screen/Cargo.tomlindex df15e4b3d..9e9e2e3ec 100644--- a/crates/screenpipe-screen/Cargo.toml+++ b/crates/screenpipe-screen/Cargo.toml@@ -19,6 +19,7 @@ default = []e2e = [][dependencies]+screenpipe-fs = { path = "../screenpipe-fs" }screenpipe-a11y = { path = "../screenpipe-a11y" }serde_json = { workspace = true }chrono = { workspace = true }diff --git a/crates/screenpipe-screen/src/snapshot_writer.rs b/crates/screenpipe-screen/src/snapshot_writer.rsindex 59ed30a7a..bc48062a1 100644--- a/crates/screenpipe-screen/src/snapshot_writer.rs+++ b/crates/screenpipe-screen/src/snapshot_writer.rs@@ -90,7 +90,7 @@ impl SnapshotWriter {let mut encoder = JpegEncoder::new_with_quality(&mut writer, quality);encoder.encode_image(img)?;writer.flush()?;- writer.get_ref().sync_all()?;+ screenpipe_fs::sync_all(writer.get_ref())?;debug!("snapshot written: {} ({}x{} -> {}x{}, q={})",diff --git a/crates/screenpipe-sqlite-coordinator/Cargo.toml b/crates/screenpipe-sqlite-coordinator/Cargo.tomlindex d755d0b43..ab01a110f 100644--- a/crates/screenpipe-sqlite-coordinator/Cargo.toml+++ b/crates/screenpipe-sqlite-coordinator/Cargo.toml@@ -8,6 +8,7 @@ license-file.workspace = trueedition.workspace = true[dependencies]+screenpipe-fs = { path = "../screenpipe-fs" }fs2 = "0.4"libsqlite3-sys = { workspace = true }serde = { workspace = true }diff --git a/crates/screenpipe-sqlite-coordinator/src/quarantine.rs b/crates/screenpipe-sqlite-coordinator/src/quarantine.rsindex 6b2f28c0d..e48411aaf 100644--- a/crates/screenpipe-sqlite-coordinator/src/quarantine.rs+++ b/crates/screenpipe-sqlite-coordinator/src/quarantine.rs@@ -285,7 +285,7 @@ fn marker_for(#[cfg(unix)]fn sync_parent(path: &Path) -> io::Result<()> {if let Some(parent) = path.parent() {- fs::File::open(parent)?.sync_all()?;+ screenpipe_fs::sync_all(&fs::File::open(parent)?)?;}Ok(())}@@ -318,7 +318,7 @@ fn atomic_write_json(path: &Path, marker: &SqliteQuarantineMarker) -> io::Result.open(&temp_path)?;file.write_all(&body)?;file.write_all(b"\n")?;- file.sync_all()?;+ screenpipe_fs::sync_all(&file)?;fs::rename(&temp_path, path)?;sync_parent(path)})();diff --git a/crates/screenpipe-sqlite-coordinator/src/verification.rs b/crates/screenpipe-sqlite-coordinator/src/verification.rsindex 4ba9eed8b..3aaf79ae3 100644--- a/crates/screenpipe-sqlite-coordinator/src/verification.rs+++ b/crates/screenpipe-sqlite-coordinator/src/verification.rs@@ -105,15 +105,62 @@ pub async fn inspect_database_health(}async fn inspect_connection(path: &Path) -> Result<usize, DatabaseHealthError> {+ #[cfg(target_os = "macos")]+ let network_volume = screenpipe_fs::is_network_volume(path)+ .map_err(|e| DatabaseHealthError::Unavailable(e.into()))?;+ #[cfg(not(target_os = "macos"))]+ let network_volume = false;+ let network_wal = if network_volume {+ use std::io::Read;+ let mut header = [0_u8; 20];+ let read = std::fs::File::open(path).and_then(|mut file| file.read_exact(&mut header));+ match read {+ Ok(()) => &header[..16] == b"SQLite format 3\0" && header[18..20] == [2, 2],+ Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => false,+ Err(error) => return Err(DatabaseHealthError::Unavailable(error.into())),+ }+ } else {+ false+ };+ // The macOS network VFS cannot open a WAL through a physically read-only+ // handle. All owners are closed here; use its private exclusive WAL index+ // with a SQL write barrier and no checkpoint on close. This lets us verify+ // existing NAS histories without rewriting their database or WAL files.let options = SqliteConnectOptions::new().filename(path).create_if_missing(false)- .read_only(true)+ .read_only(!network_wal)+ .pragma("query_only", "ON")+ .pragma(+ "locking_mode",+ if network_wal { "EXCLUSIVE" } else { "NORMAL" },+ ).busy_timeout(Duration::from_secs(5)).disable_statement_logging();let mut connection = SqliteConnection::connect_with(&options).await.map_err(classify_verification_error)?;+ if network_wal {+ let mut handle = connection+ .lock_handle()+ .await+ .map_err(classify_verification_error)?;+ // SAFETY: SQLx lends its live, exclusively locked sqlite3 connection;+ // this documented boolean db_config takes an optional result pointer.+ let result = unsafe {+ libsqlite3_sys::sqlite3_db_config(+ handle.as_raw_handle().as_ptr(),+ libsqlite3_sys::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE,+ 1_i32,+ std::ptr::null_mut::<i32>(),+ )+ };+ if result != libsqlite3_sys::SQLITE_OK {+ return Err(DatabaseHealthError::Unavailable(sqlx::Error::Protocol(+ format!("cannot disable verification checkpoint: {result}"),+ )));+ }+ }let result = sqlx::query_scalar::<_, String>("PRAGMA integrity_check").fetch_all(&mut connection).await;diff --git a/docs/coverage/CORE.md b/docs/coverage/CORE.mdindex 29c415321..9e8814a57 100644--- a/docs/coverage/CORE.md+++ b/docs/coverage/CORE.md@@ -6,6 +6,13 @@ each declared platform and layer based on non-ignored Rust test blocks,confidence, and criticality.- Manifest: `docs/coverage/core-engine-map.json`+- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y, screenpipe-fs+- Mapped suites: 34+- Mapped Rust files: 359+- Active test blocks: 3524+- Ignored/manual test blocks: 148+- Declared test blocks: 3672+- Weighted coverage points: 2910.1- Tracked crates: screenpipe-engine, screenpipe-db, screenpipe-sqlite-coordinator, screenpipe-audio, screenpipe-screen, screenpipe-a11y- Mapped suites: 33- Mapped Rust files: 359@@ -23,6 +30,9 @@ are explicitly enabled in a runtime lane.| Platform | Suites | Active tests | Ignored tests | Weighted points | Layers | Flows | Critical score || --- | --- | --- | --- | --- | --- | --- | --- |+| windows | 31 | 3381 | 142 | 2845.0 | 21 | 11 | 100% |+| macos | 31 | 3443 | 122 | 2859.0 | 22 | 11 | 100% |+| linux | 27 | 3011 | 114 | 2504.3 | 20 | 11 | 100% || windows | 30 | 3380 | 141 | 2844.0 | 21 | 11 | 100% || macos | 30 | 3442 | 121 | 2858.0 | 22 | 11 | 100% || linux | 26 | 3010 | 113 | 2503.3 | 20 | 11 | 100% |@@ -32,11 +42,13 @@ are explicitly enabled in a runtime lane.| Crate | Suites | Integration files | Source unit files | Active tests | Ignored tests | Weighted points | Flows || --- | --- | --- | --- | --- | --- | --- | --- || screenpipe-engine | 11 | 20 | 119 | 1720 | 42 | 1325.1 | 10 |+| screenpipe-db | 5 | 58 | 23 | 545 | 24 | 518.3 | 9 || screenpipe-db | 5 | 59 | 23 | 547 | 23 | 520.3 | 9 || screenpipe-sqlite-coordinator | 1 | 0 | 3 | 27 | 0 | 27.0 | 2 || screenpipe-audio | 6 | 25 | 51 | 618 | 44 | 543.2 | 5 || screenpipe-screen | 6 | 9 | 18 | 257 | 9 | 232.1 | 4 || screenpipe-a11y | 4 | 2 | 30 | 354 | 29 | 261.3 | 3 |+| screenpipe-fs | 1 | 0 | 1 | 3 | 0 | 3.0 | 3 |## Line Coverage@@ -62,9 +74,10 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| Layer | windows | macos | linux || --- | --- | --- | --- || accessibility | 4 suites / 352 active / 30 ignored / 323.5 pts | 4 suites / 410 active / 10 ignored / 332.0 pts | 4 suites / 330 active / 7 ignored / 302.5 pts |-| audio | 7 suites / 731 active / 45 ignored / 656.2 pts | 7 suites / 731 active / 45 ignored / 656.2 pts | 6 suites / 656 active / 44 ignored / 603.7 pts |+| audio | 8 suites / 734 active / 45 ignored / 659.2 pts | 8 suites / 734 active / 45 ignored / 659.2 pts | 7 suites / 659 active / 44 ignored / 606.7 pts || audio-device | 2 suites / 218 active / 7 ignored / 195.5 pts | 2 suites / 218 active / 7 ignored / 195.5 pts | 1 suites / 143 active / 6 ignored / 143.0 pts || configuration | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts | 2 suites / 143 active / 3 ignored / 129.2 pts |+| database | 7 suites / 496 active / 20 ignored / 469.3 pts | 7 suites / 496 active / 20 ignored / 469.3 pts | 7 suites / 496 active / 20 ignored / 469.3 pts || database | 6 suites / 495 active / 19 ignored / 468.3 pts | 6 suites / 495 active / 19 ignored / 468.3 pts | 6 suites / 495 active / 19 ignored / 468.3 pts || db-search | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts | 2 suites / 113 active / 9 ignored / 113.0 pts || engine-lifecycle | 6 suites / 244 active / 1 ignored / 219.0 pts | 6 suites / 244 active / 1 ignored / 219.0 pts | 5 suites / 238 active / 1 ignored / 217.3 pts |@@ -72,17 +85,21 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| meeting | 6 suites / 1624 active / 20 ignored / 1325.8 pts | 6 suites / 1624 active / 20 ignored / 1325.8 pts | 4 suites / 1287 active / 16 ignored / 1011.3 pts || ocr | 4 suites / 125 active / 7 ignored / 119.0 pts | 4 suites / 129 active / 7 ignored / 124.5 pts | 3 suites / 120 active / 6 ignored / 115.5 pts || os-integration | 1 suites / 6 active / 0 ignored / 1.7 pts | 1 suites / 6 active / 0 ignored / 1.7 pts | - |+| performance | 13 suites / 1548 active / 75 ignored / 1368.2 pts | 14 suites / 1655 active / 79 ignored / 1411.0 pts | 13 suites / 1548 active / 75 ignored / 1368.2 pts || performance | 13 suites / 1550 active / 74 ignored / 1370.2 pts | 14 suites / 1657 active / 78 ignored / 1413.0 pts | 13 suites / 1550 active / 74 ignored / 1370.2 pts || pipes | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts || privacy | 5 suites / 942 active / 37 ignored / 763.8 pts | 5 suites / 1000 active / 17 ignored / 772.3 pts | 5 suites / 920 active / 14 ignored / 742.7 pts || real-app | - | 1 suites / 107 active / 4 ignored / 42.8 pts | - || speaker | 2 suites / 368 active / 9 ignored / 368.0 pts | 2 suites / 368 active / 9 ignored / 368.0 pts | 2 suites / 368 active / 9 ignored / 368.0 pts |+| storage | 4 suites / 610 active / 37 ignored / 504.1 pts | 4 suites / 610 active / 37 ignored / 504.1 pts | 4 suites / 610 active / 37 ignored / 504.1 pts |+| sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts |+| timeline | 4 suites / 1157 active / 41 ignored / 944.3 pts | 4 suites / 1157 active / 41 ignored / 944.3 pts | 4 suites / 1157 active / 41 ignored / 944.3 pts || storage | 4 suites / 612 active / 36 ignored / 506.1 pts | 4 suites / 612 active / 36 ignored / 506.1 pts | 4 suites / 612 active / 36 ignored / 506.1 pts || sync | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts | 1 suites / 511 active / 3 ignored / 357.7 pts || timeline | 4 suites / 1159 active / 40 ignored / 946.3 pts | 4 suites / 1159 active / 40 ignored / 946.3 pts | 4 suites / 1159 active / 40 ignored / 946.3 pts || transcription | 5 suites / 808 active / 41 ignored / 633.3 pts | 5 suites / 808 active / 41 ignored / 633.3 pts | 5 suites / 808 active / 41 ignored / 633.3 pts || ui-events | 4 suites / 758 active / 29 ignored / 576.2 pts | 3 suites / 709 active / 5 ignored / 541.9 pts | 3 suites / 709 active / 5 ignored / 541.9 pts |-| vision-capture | 5 suites / 549 active / 32 ignored / 433.8 pts | 5 suites / 553 active / 32 ignored / 439.3 pts | 4 suites / 544 active / 31 ignored / 430.3 pts |+| vision-capture | 6 suites / 552 active / 32 ignored / 436.8 pts | 6 suites / 556 active / 32 ignored / 442.3 pts | 5 suites / 547 active / 31 ignored / 433.3 pts |## Critical Flow Matrix@@ -132,6 +149,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-audio-meetings-speakers | screenpipe-db | windows, macos, linux | database, audio, meeting, speaker | audio-record-transcribe, audio-device-health, meeting-live-notes | high | strong | integration | 15 | 113 | 1 | Audio transcript dedupe, live meeting mirroring, end-generation and open-meeting invariants, liveness, and speaker reassignment coverage. || db-runtime-reliability | screenpipe-db | windows, macos, linux | database, performance | performance-liveness | high | partial | mixed | 15 | 43 | 6 | SQLite hard-fault classification, failpoint VFS injection, fresh-identity recovery verification with integrity/FK/write canaries, query cancellation, close-severs-connections regressions, multi-pool WAL parity, runtime version pinning, and WAL chaos plus memory-pressure probes, plus read-only quarantine self-heal verification for transient IOERR faults. || db-search-indexing | screenpipe-db | windows, macos, linux | db-search, ocr, accessibility, performance | local-api-search, capture-ocr-pipeline, accessibility-ui-events, performance-liveness | high | strong | mixed | 13 | 105 | 4 | FTS, tokenizer, OCR snapshot search, query planning, ordering, accessibility search, and contention coverage. |+| db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 31 | 257 | 11 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || db-timeline-frames | screenpipe-db | windows, macos, linux | database, timeline, storage, performance | timeline-streaming, performance-liveness | high | strong | mixed | 32 | 259 | 10 | Frame/audio joins, timeline query shape, suggestions frames, write queue, DB primitives (src/db.rs split into src/db/ modules), feedback record upserts, media eviction anti-join regressions, SAF output registry, semantic storage, and timeline performance. || engine-api-routes | screenpipe-engine | windows, macos, linux | local-api, timeline, meeting, transcription | local-api-search, timeline-streaming, meeting-live-notes, audio-record-transcribe | high | partial | mixed | 40 | 408 | 4 | Route/unit coverage for search, health, streaming, meetings, time/timezone, and transcription. Legacy endpoint/websocket tests require local data and remain ignored. || engine-capture-timeline | screenpipe-engine | windows, macos, linux | vision-capture, timeline, storage, performance | capture-ocr-pipeline, timeline-streaming, performance-liveness | high | partial | mixed | 25 | 301 | 26 | Covers capture trigger logic, frame/audio linking, hot cache, timeline refresh regressions, fragmented MP4 extraction, and HD-mode control. Several real-data tests are intentionally ignored by default. |@@ -144,6 +162,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-retention-storage | screenpipe-engine | windows, macos, linux | storage, engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | mixed | 5 | 39 | 0 | Local retention deletion (including lean-mode heavy-text stripping), cloud archive watermarking, atomic state-file replacement, and the low-disk capture monitor. || engine-telemetry-observability | screenpipe-engine | windows, macos, linux | engine-lifecycle, performance | engine-health-lifecycle, performance-liveness | medium | strong | unit | 6 | 30 | 0 | PostHog capture gating, telemetry context shaping, piggyback telemetry forwarding, crash-log helpers, resource monitoring, and the recording-coverage reliability metric. || engine-workflows | screenpipe-engine | windows, macos, linux | local-api, storage | local-api-search | high | partial | mixed | 4 | 13 | 0 | Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality. |+| filesystem-durable-sync | screenpipe-fs | macos, linux, windows | database, vision-capture, audio | capture-ocr-pipeline, audio-record-transcribe, engine-health-lifecycle | high | strong | unit | 1 | 3 | 0 | Durable flushes preserve local full-sync behavior, fall back to fsync only for macOS ENOTSUP, and propagate real I/O or fallback failures. Real SMB migration and continued recording are covered by the marked-volume integration test. || screen-capture-ocr-contract | screenpipe-screen | windows, macos, linux | vision-capture, ocr | capture-ocr-pipeline | high | partial | unit | 3 | 15 | 0 | Cross-platform cached-OCR unit coverage for RawCaptureResult to CaptureResult metadata, browser URL, focus state, window-to-screen OCR coordinate transformation, Tesseract output parsing, and contour-based text-region detection for the meeting OCR gate. || screen-capture-windowing | screenpipe-screen | windows, macos, linux | vision-capture, timeline, performance, privacy | capture-ocr-pipeline, timeline-streaming, privacy-and-redaction, performance-liveness | high | strong | mixed | 14 | 191 | 0 | Window filtering, empty-window regressions, retry policy, URL timing, monitor cache, OCR cache, snapshots, and image comparison. || screen-custom-ocr | screenpipe-screen | windows, macos, linux | ocr | capture-ocr-pipeline | medium | conditional | manual | 1 | 0 | 2 | Custom OCR tests are ignored by default and only contribute when explicitly run. |@@ -280,7 +299,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-timeline-frames | screenpipe-db | src/storage/diagnostics.rs | source | 6 | 0 | 6 || db-timeline-frames | screenpipe-db | src/storage/lifecycle.rs | source | 1 | 0 | 1 || db-runtime-reliability | screenpipe-db | src/storage/reader.rs | source | 4 | 0 | 4 |-| db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 2 | 1 | 3 |+| db-timeline-frames | screenpipe-db | src/storage/reclaim.rs | source | 3 | 1 | 4 || db-timeline-frames | screenpipe-db | src/storage/schema.rs | source | 3 | 1 | 4 || db-timeline-frames | screenpipe-db | src/storage/sealing.rs | source | 1 | 0 | 1 || db-search-indexing | screenpipe-db | src/text_normalizer.rs | source | 19 | 0 | 19 |@@ -305,6 +324,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| db-search-indexing | screenpipe-db | tests/fts_dots_test.rs | integration | 13 | 0 | 13 || db-accessibility-ui-events | screenpipe-db | tests/heavy_read_test.rs | integration | 0 | 2 | 2 || db-timeline-frames | screenpipe-db | tests/hybrid_storage.rs | integration | 16 | 0 | 16 |+| db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 15 | 5 | 20 || db-timeline-frames | screenpipe-db | tests/in_place_migration.rs | integration | 16 | 4 | 20 || db-search-indexing | screenpipe-db | tests/keyword_search_accessibility_test.rs | integration | 7 | 0 | 7 || db-search-indexing | screenpipe-db | tests/keyword_search_order_test.rs | integration | 3 | 0 | 3 |@@ -485,6 +505,7 @@ bun run coverage:core -- --llvm-cov-summary ../../docs/coverage/core-llvm-cov-su| engine-capture-timeline | screenpipe-engine | tests/video_utils_test.rs | integration | 1 | 3 | 4 || engine-api-routes | screenpipe-engine | tests/websockets_test.rs | integration | 0 | 1 | 1 || engine-workflows | screenpipe-engine | tests/workflow_catalog_contract_test.rs | integration | 2 | 0 | 2 |+| filesystem-durable-sync | screenpipe-fs | src/lib.rs | source | 3 | 0 | 3 || screen-macos-ocr | screenpipe-screen | src/apple.rs | source | 7 | 1 | 8 || screen-capture-windowing | screenpipe-screen | src/browser_utils/mod.rs | source | 13 | 0 | 13 || screen-capture-windowing | screenpipe-screen | src/capture_screenshot_by_window.rs | source | 70 | 0 | 70 |diff --git a/docs/coverage/core-engine-map.json b/docs/coverage/core-engine-map.jsonindex 0007b2eec..e07e7126b 100644--- a/docs/coverage/core-engine-map.json+++ b/docs/coverage/core-engine-map.json@@ -61,6 +61,15 @@"sourceRoots": ["src"]+ },+ {+ "crate": "screenpipe-fs",+ "testRoots": [+ "tests"+ ],+ "sourceRoots": [+ "src"+ ]}],"criticalFlows": [@@ -1378,6 +1387,33 @@"confidence": "partial","kind": "mixed","notes": "Catalog identity and persistence, revision-scoped feedback, workflow evidence and pipeline contracts, and denial of agent-granted rollout access. Does not assert live cloud discovery quality."+ },+ {+ "id": "filesystem-durable-sync",+ "label": "Local and network filesystem durability",+ "crate": "screenpipe-fs",+ "files": [+ "src/lib.rs"+ ],+ "platforms": [+ "macos",+ "linux",+ "windows"+ ],+ "layers": [+ "database",+ "vision-capture",+ "audio"+ ],+ "flows": [+ "capture-ocr-pipeline",+ "audio-record-transcribe",+ "engine-health-lifecycle"+ ],+ "criticality": "high",+ "confidence": "strong",+ "kind": "unit",+ "notes": "Durable flushes preserve local full-sync behavior, fall back to fsync only for macOS ENOTSUP, and propagate real I/O or fallback failures. Real SMB migration and continued recording are covered by the marked-volume integration test."}]}diff --git a/docs/testing/nas-storage/audio-retranscription.json b/docs/testing/nas-storage/audio-retranscription.jsonnew file mode 100644index 000000000..5cc213361--- /dev/null+++ b/docs/testing/nas-storage/audio-retranscription.json@@ -0,0 +1,30 @@+{+ "chunks_processed": 5,+ "transcriptions": [+ {+ "old_text": null,+ "audio_chunk_id": 1,+ "new_text": ""+ },+ {+ "audio_chunk_id": 2,+ "new_text": " audio device and stored on the network drive. The history remains searchable after restarting the application. Network storage validation. The recording continues after the migration. Both the old screen history and this new audio should still be available when the application starts again.",+ "old_text": null+ },+ {+ "audio_chunk_id": 3,+ "new_text": "",+ "old_text": null+ },+ {+ "old_text": null,+ "audio_chunk_id": 4,+ "new_text": ""+ },+ {+ "audio_chunk_id": 5,+ "old_text": null,+ "new_text": ""+ }+ ]+}diff --git a/docs/testing/nas-storage/baseline-smb.txt b/docs/testing/nas-storage/baseline-smb.txtnew file mode 100644index 000000000..55a7eb66f--- /dev/null+++ b/docs/testing/nas-storage/baseline-smb.txt@@ -0,0 +1,17 @@+//guest:@192.168.64.1:1445/screenpipe on /Volumes/Screenpipe (smbfs, nodev, nosuid, noowners, mounted by admin)+Filesystem Size Used Avail Capacity iused ifree %iused Mounted on+//guest:@192.168.64.1:1445/screenpipe 487Mi 36Mi 451Mi 8% 37k 462k 7% /Volumes/Screenpipe++running 1 test++thread 'unsupported_volume_fails_before_conversion' (6328) panicked at crates/screenpipe-db/tests/in_place_migration.rs:12:10:+called `Result::unwrap()` on an `Err` value: Protocol("failed to arm SQLite verification incident for /Volumes/Screenpipe/.tmpjUMu6q/db.sqlite: Operation not supported (os error 45)")+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace+test unsupported_volume_fails_before_conversion ... FAILED++failures:++failures:+ unsupported_volume_fails_before_conversion++test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 18 filtered out; finished in 0.04sdiff --git a/docs/testing/nas-storage/connection-settings-smb.txt b/docs/testing/nas-storage/connection-settings-smb.txtnew file mode 100644index 000000000..ce70caefe--- /dev/null+++ b/docs/testing/nas-storage/connection-settings-smb.txt@@ -0,0 +1,12 @@++running 2 tests+test mcp_servers::tests::upsert_then_list_returns_entry ... ok+test mcp_servers::tests::upsert_replaces_existing_entry ... ok++test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 214 filtered out; finished in 0.08s+++running 1 test+test mcp_servers::tests::delete_removes_entry ... ok++test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 215 filtered out; finished in 0.07sdiff --git a/docs/testing/nas-storage/crash-recovery-smb.txt b/docs/testing/nas-storage/crash-recovery-smb.txtnew file mode 100644index 000000000..41f23068d--- /dev/null+++ b/docs/testing/nas-storage/crash-recovery-smb.txt@@ -0,0 +1,6 @@++running 1 test+test migration_crashes_resume_without_losing_acknowledged_records has been running for over 60 seconds+test migration_crashes_resume_without_losing_acknowledged_records ... ok++test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 15 filtered out; finished in 120.61sdiff --git a/docs/testing/nas-storage/filesystem-probe.py b/docs/testing/nas-storage/filesystem-probe.pynew file mode 100644index 000000000..1ad3680cb--- /dev/null+++ b/docs/testing/nas-storage/filesystem-probe.py@@ -0,0 +1,18 @@+# screenpipe — AI that knows everything you've seen, said, or heard+# https://screenpipe.com+import os, fcntl, struct, json+root='/Volumes/Screenpipe/Mac Mini'+os.makedirs(root,exist_ok=True)+p=root+'/probe'+with open(p,'w+b') as f:+ f.write(b'x'*1048576)+ f.flush()+ for name,op in [('file_fsync',lambda:os.fsync(f.fileno())),('file_fullfsync',lambda:fcntl.fcntl(f,51)),('hole_punch',lambda:fcntl.fcntl(f,99,struct.pack('IIqq',0,0,0,1048576)))]:+ try: op(); print(json.dumps({'operation':name,'result':'ok'}))+ except OSError as e: print(json.dumps({'operation':name,'errno':e.errno,'error':str(e)}))+fd=os.open(root,os.O_RDONLY)+try:+ os.fsync(fd); print(json.dumps({'operation':'directory_fsync','result':'ok'}))+except OSError as e: print(json.dumps({'operation':'directory_fsync','errno':e.errno,'error':str(e)}))+os.close(fd)+os.unlink(p)diff --git a/docs/testing/nas-storage/filesystem-probe.txt b/docs/testing/nas-storage/filesystem-probe.txtnew file mode 100644index 000000000..2fa09ba79--- /dev/null+++ b/docs/testing/nas-storage/filesystem-probe.txt@@ -0,0 +1,4 @@+{"operation": "file_fsync", "result": "ok"}+{"operation": "file_fullfsync", "errno": 45, "error": "[Errno 45] Operation not supported"}+{"operation": "hole_punch", "errno": 45, "error": "[Errno 45] Operation not supported"}+{"operation": "directory_fsync", "result": "ok"}diff --git a/docs/testing/nas-storage/flush-benchmark.rs b/docs/testing/nas-storage/flush-benchmark.rsnew file mode 100644index 000000000..e4b62b9dc--- /dev/null+++ b/docs/testing/nas-storage/flush-benchmark.rs@@ -0,0 +1,41 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+use std::{+ fs::OpenOptions,+ io::{Seek, SeekFrom, Write},+ time::Instant,+};+fn main() {+ let mut file = OpenOptions::new()+ .read(true)+ .write(true)+ .create_new(true)+ .open("/tmp/screenpipe-nas-evidence/sync-bench.dat")+ .unwrap();+ let data = [17_u8; 4096];+ let mut baseline = Vec::new();+ let mut candidate = Vec::new();+ for round in 0..10 {+ for patched in [round % 2 == 0, round % 2 != 0] {+ let start = Instant::now();+ for _ in 0..100 {+ file.seek(SeekFrom::Start(0)).unwrap();+ file.write_all(&data).unwrap();+ if patched {+ screenpipe_fs::sync_all(&file).unwrap()+ } else {+ file.sync_all().unwrap()+ }+ }+ (if patched {+ &mut candidate+ } else {+ &mut baseline+ })+ .push(start.elapsed().as_secs_f64() * 1000.0 / 100.0);+ }+ }+ baseline.sort_by(f64::total_cmp);+ candidate.sort_by(f64::total_cmp);+ println!("APFS 4KiB overwrite+flush: 10 alternating rounds x 100 writes; baseline median_ms={:.4} candidate median_ms={:.4}; baseline_samples={baseline:?}; candidate_samples={candidate:?}",baseline[5],candidate[5]);+}diff --git a/docs/testing/nas-storage/legacy-wal-smb.txt b/docs/testing/nas-storage/legacy-wal-smb.txtnew file mode 100644index 000000000..8c3edab50--- /dev/null+++ b/docs/testing/nas-storage/legacy-wal-smb.txt@@ -0,0 +1,6 @@++running 1 test+NAS WAL: verification preserved database and committed WAL bytes; pending incident recovered; acknowledged and new rows survive restart in rollback mode+test network_wal_verification_and_reopen_preserve_committed_rows ... ok++test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 19 filtered out; finished in 13.52sdiff --git a/docs/testing/nas-storage/local-flush-benchmark.txt b/docs/testing/nas-storage/local-flush-benchmark.txtnew file mode 100644index 000000000..63b2422a2--- /dev/null+++ b/docs/testing/nas-storage/local-flush-benchmark.txt@@ -0,0 +1 @@+APFS 4KiB overwrite+flush: 10 alternating rounds x 100 writes; baseline median_ms=4.8225 candidate median_ms=4.8559; baseline_samples=[4.5991925, 4.7272925, 4.73580042, 4.7659312499999995, 4.78956416, 4.822498749999999, 4.82365833, 4.83045, 4.840102079999999, 4.91428292]; candidate_samples=[4.68973291, 4.711666660000001, 4.719995, 4.74891042, 4.76601875, 4.85592791, 4.94688125, 5.02534417, 5.0386945800000005, 5.37141]diff --git a/docs/testing/nas-storage/migration-smb.txt b/docs/testing/nas-storage/migration-smb.txtnew file mode 100644index 000000000..45693989c--- /dev/null+++ b/docs/testing/nas-storage/migration-smb.txt@@ -0,0 +1,120 @@++running 1 test+2026-09-18T06:00:46.430413Z INFO screenpipe_db::storage::diagnostics: storage migration diagnostic event="started" diagnostic={"schema_version":1,"process_id":1334,"database_version":"0.4.50","attempt_id":"b4cfe818-06d9-43ef-ab6b-1e0e6aee74d1","kind":"conversion","started_at":"2026-09-18T06:00:46.430235+00:00","observed_at":"2026-09-18T06:00:46.430329+00:00","status":"running","stage":"starting","table":null,"first_id":null,"last_id":null,"batch_rows":null,"batch_bytes":null,"completed_records":null,"total_records":null,"table_records":{},"elapsed_seconds":0,"idle_seconds":0,"failure_stage":null,"error":null}+2026-09-18T06:00:46.456566Z INFO screenpipe_db::storage::reclaim: filesystem does not support sparse reclamation; keeping reusable SQLite free pages and enforcing migration disk reserve+2026-09-18T06:00:49.616300Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=_sqlx_migrations rows=116+2026-09-18T06:00:49.617207Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_actions rows=0+2026-09-18T06:00:49.618053Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_evidence rows=0+2026-09-18T06:00:49.619246Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_intervals rows=0+2026-09-18T06:00:49.620301Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_ledger_state rows=0+2026-09-18T06:00:49.621104Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_tasks rows=0+2026-09-18T06:00:49.622191Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=audio_chunks rows=0+2026-09-18T06:00:49.622935Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=audio_tags rows=0+2026-09-18T06:00:49.623716Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=audio_transcriptions rows=0+2026-09-18T06:00:49.624697Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=diarization_runs rows=0+2026-09-18T06:00:49.625468Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=diarization_segments rows=0+2026-09-18T06:00:49.626144Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=display_layout rows=0+2026-09-18T06:00:51.142857Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=elements rows=48+2026-09-18T06:00:51.144045Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=feedback rows=0+2026-09-18T06:00:55.962003Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=frames rows=48+2026-09-18T06:00:55.962796Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=meeting_transcript_segments rows=0+2026-09-18T06:00:55.963479Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=meetings rows=0+2026-09-18T06:00:55.964472Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=memories rows=0+2026-09-18T06:00:55.965183Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=output_search_index rows=0+2026-09-18T06:00:55.965918Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=outputs rows=0+2026-09-18T06:00:55.966870Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=pipe_event_runs rows=0+2026-09-18T06:00:55.967902Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=pipe_executions rows=0+2026-09-18T06:00:55.968659Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=pipe_scheduler_state rows=0+2026-09-18T06:00:55.969550Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_actor_aliases rows=0+2026-09-18T06:00:55.970479Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_actors rows=0+2026-09-18T06:00:55.971178Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_item_actors rows=0+2026-09-18T06:00:55.972213Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_items rows=0+2026-09-18T06:00:55.972883Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_run_items rows=0+2026-09-18T06:00:55.973560Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_runs rows=0+2026-09-18T06:00:55.974474Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=speaker_embeddings rows=0+2026-09-18T06:00:55.975196Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=speaker_identity_evidence rows=0+2026-09-18T06:00:55.975877Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=speakers rows=0+2026-09-18T06:00:55.977053Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=sqlite_sequence rows=4+2026-09-18T06:00:55.978007Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=tags rows=0+2026-09-18T06:00:55.979005Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=ui_events rows=0+2026-09-18T06:00:55.979723Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=video_chunks rows=0+2026-09-18T06:00:55.980614Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=vision_tags rows=0+2026-09-18T06:00:56.048054Z INFO screenpipe_db::storage::reclaim: filesystem does not support sparse reclamation; keeping reusable SQLite free pages and enforcing migration disk reserve+2026-09-18T06:00:56.335110Z INFO screenpipe_db::storage::bulk::schema: building bulk storage schema table="elements"+2026-09-18T06:00:56.412341Z INFO screenpipe_db::storage::bulk::schema: building bulk storage schema table="audio_transcriptions"+2026-09-18T06:00:56.483418Z INFO screenpipe_db::storage::bulk::schema: building bulk storage schema table="ui_events"+2026-09-18T06:00:56.552217Z INFO screenpipe_db::storage::bulk::schema: building bulk storage schema table="semantic_items"+2026-09-18T06:00:56.662382Z INFO screenpipe_db::storage::bulk::schema: building bulk storage schema table="pipe_executions"+2026-09-18T06:00:56.722109Z INFO screenpipe_db::storage::bulk::schema: building bulk storage schema table="meeting_transcript_segments"+2026-09-18T06:00:56.784225Z INFO screenpipe_db::storage::bulk::schema: building bulk storage schema table="outputs"+2026-09-18T06:01:16.442159Z INFO screenpipe_db::storage::diagnostics: storage migration diagnostic event="heartbeat" diagnostic={"schema_version":1,"process_id":1334,"database_version":"0.4.50","attempt_id":"b4cfe818-06d9-43ef-ab6b-1e0e6aee74d1","kind":"conversion","started_at":"2026-09-18T06:00:46.430235+00:00","observed_at":"2026-09-18T06:01:16.442065+00:00","status":"running","stage":"committing_frame_archive","table":"frames","first_id":25,"last_id":26,"batch_rows":2,"batch_bytes":983096,"completed_records":24,"total_records":96,"table_records":{"_sqlx_migrations":116,"activity_actions":0,"activity_evidence":0,"activity_intervals":0,"activity_ledger_state":0,"activity_tasks":0,"audio_chunks":0,"audio_tags":0,"audio_transcriptions":0,"diarization_runs":0,"diarization_segments":0,"display_layout":0,"elements":48,"feedback":0,"frames":48,"meeting_transcript_segments":0,"meetings":0,"memories":0,"output_search_index":0,"outputs":0,"pipe_event_runs":0,"pipe_executions":0,"pipe_scheduler_state":0,"semantic_actor_aliases":0,"semantic_actors":0,"semantic_item_actors":0,"semantic_items":0,"semantic_run_items":0,"semantic_runs":0,"speaker_embeddings":0,"speaker_identity_evidence":0,"speakers":0,"sqlite_sequence":4,"tags":0,"ui_events":0,"video_chunks":0,"vision_tags":0},"elapsed_seconds":30,"idle_seconds":0,"failure_stage":null,"error":null}+2026-09-18T06:01:26.980182Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:29.112981Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+test non_sparse_volume_migrates_and_keeps_recording_after_restart has been running for over 60 seconds+2026-09-18T06:01:31.205479Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:33.241612Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:35.169095Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:37.049472Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:38.834816Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:40.604754Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:42.292471Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:43.942755Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:45.476275Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:46.452572Z INFO screenpipe_db::storage::diagnostics: storage migration diagnostic event="heartbeat" diagnostic={"schema_version":1,"process_id":1334,"database_version":"0.4.50","attempt_id":"b4cfe818-06d9-43ef-ab6b-1e0e6aee74d1","kind":"conversion","started_at":"2026-09-18T06:00:46.430235+00:00","observed_at":"2026-09-18T06:01:46.452480+00:00","status":"running","stage":"committing_element_staging","table":"elements","first_id":23,"last_id":24,"batch_rows":null,"batch_bytes":null,"completed_records":70,"total_records":96,"table_records":{"_sqlx_migrations":116,"activity_actions":0,"activity_evidence":0,"activity_intervals":0,"activity_ledger_state":0,"activity_tasks":0,"audio_chunks":0,"audio_tags":0,"audio_transcriptions":0,"diarization_runs":0,"diarization_segments":0,"display_layout":0,"elements":48,"feedback":0,"frames":48,"meeting_transcript_segments":0,"meetings":0,"memories":0,"output_search_index":0,"outputs":0,"pipe_event_runs":0,"pipe_executions":0,"pipe_scheduler_state":0,"semantic_actor_aliases":0,"semantic_actors":0,"semantic_item_actors":0,"semantic_items":0,"semantic_run_items":0,"semantic_runs":0,"speaker_embeddings":0,"speaker_identity_evidence":0,"speakers":0,"sqlite_sequence":4,"tags":0,"ui_events":0,"video_chunks":0,"vision_tags":0},"elapsed_seconds":60,"idle_seconds":0,"failure_stage":null,"error":null}+2026-09-18T06:01:47.009144Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:48.461330Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:49.858574Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:51.153947Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:52.432764Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:53.658131Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:54.827932Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:55.938619Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:56.973103Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:57.947561Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:58.862975Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:01:59.745989Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:02:00.541019Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=2 files=1+2026-09-18T06:02:02.075869Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=_sqlx_migrations rows=116+2026-09-18T06:02:02.079246Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_actions rows=0+2026-09-18T06:02:02.082644Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_evidence rows=0+2026-09-18T06:02:02.086292Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_intervals rows=0+2026-09-18T06:02:02.089649Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_ledger_state rows=0+2026-09-18T06:02:02.093040Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=activity_tasks rows=0+2026-09-18T06:02:02.096691Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=audio_chunks rows=0+2026-09-18T06:02:02.099976Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=audio_tags rows=0+2026-09-18T06:02:02.103450Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=audio_transcriptions rows=0+2026-09-18T06:02:02.107117Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=diarization_runs rows=0+2026-09-18T06:02:02.110480Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=diarization_segments rows=0+2026-09-18T06:02:02.113771Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=display_layout rows=0+2026-09-18T06:02:02.188055Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=elements rows=48+2026-09-18T06:02:02.191769Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=feedback rows=0+2026-09-18T06:02:02.449357Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=frames rows=48+2026-09-18T06:02:02.453649Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=meeting_transcript_segments rows=0+2026-09-18T06:02:02.458026Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=meetings rows=0+2026-09-18T06:02:02.461979Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=memories rows=0+2026-09-18T06:02:02.465858Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=output_search_index rows=0+2026-09-18T06:02:02.470358Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=outputs rows=0+2026-09-18T06:02:02.474252Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=pipe_event_runs rows=0+2026-09-18T06:02:02.478534Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=pipe_executions rows=0+2026-09-18T06:02:02.482735Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=pipe_scheduler_state rows=0+2026-09-18T06:02:02.486688Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_actor_aliases rows=0+2026-09-18T06:02:02.490590Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_actors rows=0+2026-09-18T06:02:02.494900Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_item_actors rows=0+2026-09-18T06:02:02.499017Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_items rows=0+2026-09-18T06:02:02.503078Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_run_items rows=0+2026-09-18T06:02:02.507329Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=semantic_runs rows=0+2026-09-18T06:02:02.511258Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=speaker_embeddings rows=0+2026-09-18T06:02:02.515427Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=speaker_identity_evidence rows=0+2026-09-18T06:02:02.519906Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=speakers rows=0+2026-09-18T06:02:02.526502Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=sqlite_sequence rows=4+2026-09-18T06:02:02.530702Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=tags rows=0+2026-09-18T06:02:02.535084Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=ui_events rows=0+2026-09-18T06:02:02.539451Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=video_chunks rows=0+2026-09-18T06:02:02.543426Z INFO screenpipe_db::storage::lifecycle: storage parity receipt complete table=vision_tags rows=0+2026-09-18T06:02:02.749744Z INFO screenpipe_db::storage::diagnostics: storage migration diagnostic event="completed" diagnostic={"schema_version":1,"process_id":1334,"database_version":"0.4.50","attempt_id":"b4cfe818-06d9-43ef-ab6b-1e0e6aee74d1","kind":"conversion","started_at":"2026-09-18T06:00:46.430235+00:00","observed_at":"2026-09-18T06:02:02.749644+00:00","status":"completed","stage":"saving_completion_receipt","table":"vision_tags","first_id":-9223372036854775808,"last_id":null,"batch_rows":0,"batch_bytes":null,"completed_records":96,"total_records":96,"table_records":{"_sqlx_migrations":116,"activity_actions":0,"activity_evidence":0,"activity_intervals":0,"activity_ledger_state":0,"activity_tasks":0,"audio_chunks":0,"audio_tags":0,"audio_transcriptions":0,"diarization_runs":0,"diarization_segments":0,"display_layout":0,"elements":48,"feedback":0,"frames":48,"meeting_transcript_segments":0,"meetings":0,"memories":0,"output_search_index":0,"outputs":0,"pipe_event_runs":0,"pipe_executions":0,"pipe_scheduler_state":0,"semantic_actor_aliases":0,"semantic_actors":0,"semantic_item_actors":0,"semantic_items":0,"semantic_run_items":0,"semantic_runs":0,"speaker_embeddings":0,"speaker_identity_evidence":0,"speakers":0,"sqlite_sequence":4,"tags":0,"ui_events":0,"video_chunks":0,"vision_tags":0},"elapsed_seconds":76,"idle_seconds":0,"failure_stage":null,"error":null}+2026-09-18T06:02:03.618910Z INFO screenpipe_db::storage::bulk::elements::lifecycle: sealed element records rows=1 files=1+2026-09-18T06:02:03.628762Z INFO screenpipe_db::storage::bulk::lifecycle: sealing bulk payload batch table="audio_transcriptions" rows=1+non-sparse migration: 48 historical frames verified; receipt present; journal removed; new frame, element and transcript sealed, searchable and intact after restart; source_bytes=48115712 index_bytes=48881664 parquet_bytes=154176+test non_sparse_volume_migrates_and_keeps_recording_after_restart ... ok++test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 18 filtered out; finished in 93.62sdiff --git a/docs/testing/nas-storage/native-after-stall-health.json b/docs/testing/nas-storage/native-after-stall-health.jsonnew file mode 100644index 000000000..2726427d9--- /dev/null+++ b/docs/testing/nas-storage/native-after-stall-health.json@@ -0,0 +1,136 @@+{+ "status": "degraded",+ "status_code": 503,+ "last_frame_timestamp": "2026-09-18T06:31:28+00:00",+ "last_audio_timestamp": "2026-09-18T06:27:16+00:00",+ "frame_status": "ok",+ "vision_reason": "ok",+ "loop_stage": "trigger-wait",+ "loop_stage_age_secs": 1,+ "audio_capture_mode": "always",+ "audio_status": "not_started",+ "message": "some systems are not healthy: audio",+ "verbose_instructions": "Audio system is not working properly. Check if microphone permissions are enabled and devices are connected. If audio chunks are being dropped, try switching to a smaller Whisper model or using cloud transcription.\n",+ "device_status_details": "System Audio (output): inactive (last activity: 261s ago), Apple Virtual Sound Device (input): inactive (last activity: 261s ago)",+ "capture_status": {+ "status": "audio_not_started",+ "severity": "warning",+ "reason": "audio capture has not produced data yet",+ "audio_disabled": false,+ "active_audio_devices": 2,+ "paused_audio_devices": 0+ },+ "monitors": [+ "Display 1 (1024x768)"+ ],+ "pipeline": {+ "uptime_secs": 310.418954917,+ "frames_captured": 12,+ "frames_db_written": 12,+ "frames_dropped": 0,+ "frame_drop_rate": 0.0,+ "frames_dropped_timeout": 0,+ "frames_dropped_error": 0,+ "silent_loss": 0,+ "silent_loss_rate": 0.0,+ "capture_attempts": 16,+ "capture_loop_heartbeats": 1168,+ "dedup_skips": 4,+ "frames_corrupt_black": 0,+ "frames_corrupt_green": 0,+ "last_capture_attempt_ts": 1789713079,+ "last_capture_loop_heartbeat_ts": 1789713096,+ "capture_fps_actual": 0.038657433155808954,+ "avg_ocr_latency_ms": 855.5,+ "avg_db_latency_ms": 1091.8333333333333,+ "ocr_queue_depth": 0,+ "video_queue_depth": 0,+ "time_to_first_frame_ms": 2288.311,+ "pipeline_stall_count": 0,+ "ocr_cache_hit_rate": 0.3333333333333333,+ "ocr_empty": 0+ },+ "audio_pipeline": {+ "uptime_secs": 310.418965333,+ "chunks_sent": 20,+ "chunks_channel_full": 0,+ "stream_timeouts": 0,+ "vad_passed": 0,+ "vad_rejected": 19,+ "vad_passthrough_rate": 0.0,+ "avg_speech_ratio": 0.0,+ "transcriptions_completed": 0,+ "transcriptions_empty": 0,+ "transcription_errors": 0,+ "db_inserted": 0,+ "total_words": 0,+ "words_per_minute": 0.0,+ "chunks_received": 20,+ "process_errors": 0,+ "audio_level_rms": 0.0,+ "per_device_audio_level_rms": {+ "Apple Virtual Sound Device (input)": 0.0,+ "System Audio (output)": 0.0+ },+ "audio_devices": [+ "System Audio (output)",+ "Apple Virtual Sound Device (input)"+ ],+ "transcription_mode": "batch",+ "transcription_paused": false,+ "meeting_detected": false+ },+ "accessibility": {+ "walks_total": 21,+ "walks_stored": 12,+ "walks_deduped": 4,+ "walks_empty": 5,+ "walks_error": 0,+ "walks_worker_timeout": 0,+ "walks_worker_saturated": 0,+ "worker_restarts": 0,+ "walks_truncated": 0,+ "walks_truncated_timeout": 0,+ "walks_truncated_max_nodes": 0,+ "truncation_rate": 0.0,+ "avg_walk_duration_ms": 33,+ "max_walk_duration_ms": 71,+ "avg_nodes_per_walk": 9,+ "max_depth_reached": 3,+ "total_text_chars": 4493+ },+ "ui_recorder": {+ "configured": true,+ "running": true,+ "mode": "full",+ "clipboard_capture": false,+ "input_tap_running": true,+ "app_events_running": true,+ "events_inserted": 12,+ "last_event_at": "2026-09-18T06:30:16+00:00"+ },+ "recording_coverage": {+ "recording_secs": 388,+ "active_stalled_secs": 0,+ "active_paused_secs": 0,+ "idle_secs": 41,+ "active_secs": 388,+ "coverage_ratio": 1.0+ },+ "pool_stats": {+ "read_pool_size": 5,+ "read_pool_idle": 5,+ "write_pool_size": 2,+ "write_pool_idle": 2+ },+ "write_queue_degraded": false,+ "write_queue_consecutive_fatal": 0,+ "write_queue_consecutive_contention": 0,+ "write_pool_reopens": 0,+ "persistent_failure_signals": 0,+ "vision_db_write_stalled": false,+ "audio_db_write_stalled": false,+ "drm_content_paused": false,+ "schedule_paused": false,+ "version": "0.4.50"+}diff --git a/docs/testing/nas-storage/native-completion-receipt.json b/docs/testing/nas-storage/native-completion-receipt.jsonnew file mode 100644index 000000000..42fe87132--- /dev/null+++ b/docs/testing/nas-storage/native-completion-receipt.json@@ -0,0 +1,198 @@+{+ "database_id": "0ead31f8-872a-4b45-9f9f-d6bbe527c919",+ "generation": "d9ae4f35-2564-4f49-a825-0a7fdeb20bfc",+ "frames": 23,+ "tables": [+ {+ "table": "_sqlx_migrations",+ "rows": 116,+ "sha256": "5f1f93ad6802725c08762ee80badf5f6f28d0680846563eab299b1d3edac3a61"+ },+ {+ "table": "activity_actions",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "activity_evidence",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "activity_intervals",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "activity_ledger_state",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "activity_tasks",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "audio_chunks",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "audio_tags",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "audio_transcriptions",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "diarization_runs",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "diarization_segments",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "display_layout",+ "rows": 1,+ "sha256": "6276b802984af91edb9c3ec8a54650baaef2da8f83b99c3d04cefd554e56e4c8"+ },+ {+ "table": "elements",+ "rows": 302,+ "sha256": "11045bd6236efea46095f2d506ad7d80f801f0193d3ee063f51b5f04b52d1f99"+ },+ {+ "table": "feedback",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "frames",+ "rows": 23,+ "sha256": "9396d546abeea2e80ffb5824aaaef2af1184a2ec154818f3740ebf8f97ba6663"+ },+ {+ "table": "meeting_transcript_segments",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "meetings",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "memories",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "output_search_index",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "outputs",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "pipe_event_runs",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "pipe_executions",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "pipe_scheduler_state",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "semantic_actor_aliases",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "semantic_actors",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "semantic_item_actors",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "semantic_items",+ "rows": 1,+ "sha256": "a14be6afec791ac3990e16254b06873ce46efaa30b3f781f2ee5129feeaaef6b"+ },+ {+ "table": "semantic_run_items",+ "rows": 1,+ "sha256": "25345362ca16e3e36f0f395b26e2d95e18946b3cb07db41f7f496e66f224a763"+ },+ {+ "table": "semantic_runs",+ "rows": 1,+ "sha256": "117b0fffddd03aa2ff0510c2e047ff5f6e710b91262355031a479988e17aec52"+ },+ {+ "table": "speaker_embeddings",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "speaker_identity_evidence",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "speakers",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "sqlite_sequence",+ "rows": 9,+ "sha256": "ea682e970a7bac268eae1cdbf8a673616d8223bb396069221c0d73a1aeb6d638"+ },+ {+ "table": "tags",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ {+ "table": "ui_events",+ "rows": 6,+ "sha256": "7f28a2df41ddd8d0eb7e9e08b013064770df00dd157024673a632670e39b353a"+ },+ {+ "table": "video_chunks",+ "rows": 1,+ "sha256": "877363632c18993c585851d9cc2e5ed220b5399d00cfea095d258b170840d196"+ },+ {+ "table": "vision_tags",+ "rows": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ }+ ],+ "source_bytes": 6037504,+ "index_bytes": 6901760,+ "payload_bytes": 33814,+ "source_identity": null,+ "allocated_before_bytes": 6037504,+ "allocated_after_bytes": 6944768+}diff --git a/docs/testing/nas-storage/native-history-fixture.rs b/docs/testing/nas-storage/native-history-fixture.rsnew file mode 100644index 000000000..157a93bc6--- /dev/null+++ b/docs/testing/nas-storage/native-history-fixture.rs@@ -0,0 +1,48 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+use screenpipe_db::DatabaseManager;+#[tokio::main]+async fn main() -> anyhow::Result<()> {+ let root = std::path::PathBuf::from(std::env::args().nth(1).expect("fixture path"));+ anyhow::ensure!(+ root.parent()+ .unwrap()+ .join(".screenpipe-disposable-volume")+ .is_file(),+ "owned test share required"+ );+ std::fs::create_dir_all(&root)?;+ anyhow::ensure!(!root.join("db.sqlite").exists(), "must be fresh fixture");+ let image_path = root.join("historical-marker.jpg");+ image::DynamicImage::new_rgb8(640, 480).save(&image_path)?;+ screenpipe_fs::sync_all(&std::fs::File::open(&image_path)?)?;+ let db =+ DatabaseManager::new(root.join("db.sqlite").to_str().unwrap(), Default::default()).await?;+ db.insert_video_chunk(image_path.to_str().unwrap(), "Display 1")+ .await?;+ for n in 0..8 {+ let frame = db+ .insert_frame(+ "Display 1",+ Some(chrono::Utc::now() - chrono::Duration::minutes(20 - n)),+ None,+ Some("NAS History"),+ Some("NAS retained historical marker"),+ true,+ Some(0),+ )+ .await?;+ let mut tx = db.begin_immediate_with_retry().await?;+ sqlx::query("UPDATE frames SET full_text=?,accessibility_text=?,accessibility_tree_json=? WHERE id=?")+ .bind(format!("NAS retained historical marker {n}"))+ .bind(format!("NAS retained historical marker {n}"))+ .bind("historical payload ".repeat(32768)).bind(frame).execute(&mut **tx.conn()).await?;+ tx.commit().await?;+ }+ db.close().await;+ println!(+ "Created 8 legacy history frames through the production writer in {}",+ root.display()+ );+ Ok(())+}diff --git a/docs/testing/nas-storage/native-migration-status.json b/docs/testing/nas-storage/native-migration-status.jsonnew file mode 100644index 000000000..7b771ab3b--- /dev/null+++ b/docs/testing/nas-storage/native-migration-status.json@@ -0,0 +1,42 @@+[+ {+ "bytes_saved": 0,+ "root": "/Volumes/ScreenpipeApp/Mac Mini",+ "elapsed_seconds": 19,+ "total_records": null,+ "recovering": false,+ "message": "Your history has been migrated and recording has resumed.",+ "completed_records": null,+ "busy": false,+ "available_bytes": 31909462016,+ "completed": true,+ "error": null+ },+ {+ "can_cancel": false,+ "message": "",+ "generation": "d9ae4f35-2564-4f49-a825-0a7fdeb20bfc",+ "can_delete_source": false,+ "busy": false,+ "error": null,+ "completed": true,+ "using_new_storage": true,+ "pending": false,+ "available_bytes": 31919730688,+ "app_session_id": "f8935253-7e97-413b-b3b1-997504a4040e",+ "in_place": true,+ "source_bytes": 0,+ "root": "/Volumes/ScreenpipeApp/Mac Mini",+ "blocked_reason": null,+ "bytes_saved": 0,+ "migrated_bytes": 6944768,+ "can_migrate": false+ },+ {+ "phase": "ready",+ "message": null,+ "sinceEpochSecs": 1789712704,+ "error": null,+ "cpuCompatMode": false+ }+]diff --git a/docs/testing/nas-storage/native-restart-health.json b/docs/testing/nas-storage/native-restart-health.jsonnew file mode 100644index 000000000..81ff77fe7--- /dev/null+++ b/docs/testing/nas-storage/native-restart-health.json@@ -0,0 +1,139 @@+{+ "status": "degraded",+ "status_code": 503,+ "last_frame_timestamp": "2026-09-18T06:39:52+00:00",+ "last_audio_timestamp": "2026-09-18T06:32:53+00:00",+ "frame_status": "ok",+ "vision_reason": "ok",+ "loop_stage": "trigger-wait",+ "loop_stage_age_secs": 0,+ "audio_capture_mode": "always",+ "audio_status": "not_started",+ "message": "some systems are not healthy: audio",+ "verbose_instructions": "Audio system is not working properly. Check if microphone permissions are enabled and devices are connected. If audio chunks are being dropped, try switching to a smaller Whisper model or using cloud transcription.\n",+ "device_status_details": "Apple Virtual Sound Device (input): inactive (last activity: 427s ago), System Audio (output): inactive (last activity: 427s ago)",+ "capture_status": {+ "status": "audio_not_started",+ "severity": "warning",+ "reason": "audio capture has not produced data yet",+ "audio_disabled": false,+ "active_audio_devices": 2,+ "paused_audio_devices": 0,+ "pending_transcription_segments": 4+ },+ "monitors": [+ "Display 1 (1024x768)"+ ],+ "pipeline": {+ "uptime_secs": 427.724931042,+ "frames_captured": 18,+ "frames_db_written": 18,+ "frames_dropped": 0,+ "frame_drop_rate": 0.0,+ "frames_dropped_timeout": 0,+ "frames_dropped_error": 0,+ "silent_loss": 0,+ "silent_loss_rate": 0.0,+ "capture_attempts": 18,+ "capture_loop_heartbeats": 1655,+ "dedup_skips": 0,+ "frames_corrupt_black": 0,+ "frames_corrupt_green": 0,+ "last_capture_attempt_ts": 1789713591,+ "last_capture_loop_heartbeat_ts": 1789713600,+ "capture_fps_actual": 0.04208312093510516,+ "avg_ocr_latency_ms": 900.0,+ "avg_db_latency_ms": 422.72222222222223,+ "ocr_queue_depth": 0,+ "video_queue_depth": 0,+ "time_to_first_frame_ms": 2278.023,+ "pipeline_stall_count": 0,+ "ocr_cache_hit_rate": 0.5454545454545454,+ "ocr_empty": 0+ },+ "audio_pipeline": {+ "uptime_secs": 427.724948459,+ "chunks_sent": 28,+ "chunks_channel_full": 0,+ "stream_timeouts": 0,+ "vad_passed": 0,+ "vad_rejected": 28,+ "vad_passthrough_rate": 0.0,+ "avg_speech_ratio": 0.0,+ "transcriptions_completed": 0,+ "transcriptions_empty": 0,+ "transcription_errors": 0,+ "db_inserted": 0,+ "total_words": 0,+ "words_per_minute": 0.0,+ "chunks_received": 28,+ "process_errors": 0,+ "audio_level_rms": 0.0,+ "per_device_audio_level_rms": {+ "System Audio (output)": 0.0,+ "Apple Virtual Sound Device (input)": 0.0+ },+ "audio_devices": [+ "Apple Virtual Sound Device (input)",+ "System Audio (output)"+ ],+ "transcription_mode": "batch",+ "transcription_paused": false,+ "pending_transcription_segments": 4,+ "oldest_pending_transcription_at": "2026-09-18T06:29:00+00:00",+ "meeting_detected": false+ },+ "accessibility": {+ "walks_total": 17,+ "walks_stored": 7,+ "walks_deduped": 0,+ "walks_empty": 10,+ "walks_error": 0,+ "walks_worker_timeout": 0,+ "walks_worker_saturated": 0,+ "worker_restarts": 0,+ "walks_truncated": 0,+ "walks_truncated_timeout": 0,+ "walks_truncated_max_nodes": 0,+ "truncation_rate": 0.0,+ "avg_walk_duration_ms": 12,+ "max_walk_duration_ms": 46,+ "avg_nodes_per_walk": 5,+ "max_depth_reached": 3,+ "total_text_chars": 2317+ },+ "ui_recorder": {+ "configured": true,+ "running": true,+ "mode": "full",+ "clipboard_capture": false,+ "input_tap_running": true,+ "app_events_running": true,+ "events_inserted": 4,+ "last_event_at": "2026-09-18T06:36:18+00:00"+ },+ "recording_coverage": {+ "recording_secs": 379,+ "active_stalled_secs": 0,+ "active_paused_secs": 0,+ "idle_secs": 0,+ "active_secs": 379,+ "coverage_ratio": 1.0+ },+ "pool_stats": {+ "read_pool_size": 5,+ "read_pool_idle": 5,+ "write_pool_size": 2,+ "write_pool_idle": 2+ },+ "write_queue_degraded": false,+ "write_queue_consecutive_fatal": 0,+ "write_queue_consecutive_contention": 0,+ "write_pool_reopens": 0,+ "persistent_failure_signals": 0,+ "vision_db_write_stalled": false,+ "audio_db_write_stalled": false,+ "drm_content_paused": false,+ "schedule_paused": false,+ "version": "0.4.50"+}diff --git a/docs/testing/nas-storage/native-restart-search.json b/docs/testing/nas-storage/native-restart-search.jsonnew file mode 100644index 000000000..a9c2d73e2--- /dev/null+++ b/docs/testing/nas-storage/native-restart-search.json@@ -0,0 +1,171 @@+{+ "observed_after_app_restart": true,+ "app_restart_utc": "2026-09-18T06:32:53Z",+ "source": "Authenticated local API; only synthetic fixture content retained in this evidence",+ "retained-history": {+ "pagination": {+ "limit": 10,+ "offset": 0,+ "total": 8+ },+ "data": [+ {+ "frame_id": 8,+ "timestamp": "2026-09-18T05:46:10.851684+00:00",+ "app_name": "NAS History",+ "text": "NAS retained historical marker 7",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/historical-marker.jpg"+ },+ {+ "frame_id": 7,+ "timestamp": "2026-09-18T05:45:10.736280+00:00",+ "app_name": "NAS History",+ "text": "NAS retained historical marker 6",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/historical-marker.jpg"+ },+ {+ "frame_id": 6,+ "timestamp": "2026-09-18T05:44:10.620499+00:00",+ "app_name": "NAS History",+ "text": "NAS retained historical marker 5",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/historical-marker.jpg"+ },+ {+ "frame_id": 5,+ "timestamp": "2026-09-18T05:43:10.506151+00:00",+ "app_name": "NAS History",+ "text": "NAS retained historical marker 4",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/historical-marker.jpg"+ },+ {+ "frame_id": 4,+ "timestamp": "2026-09-18T05:42:10.393216+00:00",+ "app_name": "NAS History",+ "text": "NAS retained historical marker 3",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/historical-marker.jpg"+ },+ {+ "frame_id": 3,+ "timestamp": "2026-09-18T05:41:10.264860+00:00",+ "app_name": "NAS History",+ "text": "NAS retained historical marker 2",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/historical-marker.jpg"+ },+ {+ "frame_id": 2,+ "timestamp": "2026-09-18T05:40:10.134791+00:00",+ "app_name": "NAS History",+ "text": "NAS retained historical marker 1",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/historical-marker.jpg"+ },+ {+ "frame_id": 1,+ "timestamp": "2026-09-18T05:39:09.985239+00:00",+ "app_name": "NAS History",+ "text": "NAS retained historical marker 0",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/historical-marker.jpg"+ }+ ]+ },+ "retained-new": {+ "pagination": {+ "limit": 50,+ "offset": 0,+ "total": 11+ },+ "data": [+ {+ "frame_id": 46,+ "timestamp": "2026-09-18T06:35:59.034467+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NASNETWORKRETURN NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713359034_m1.jpg"+ },+ {+ "frame_id": 45,+ "timestamp": "2026-09-18T06:35:28.859057+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NASNETWORKRETURN NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713328859_m1.jpg"+ },+ {+ "frame_id": 44,+ "timestamp": "2026-09-18T06:34:58.460627+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NASNETWORKRETURN NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713298460_m1.jpg"+ },+ {+ "frame_id": 43,+ "timestamp": "2026-09-18T06:34:28.016114+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NASNETWORKRETURN NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713268016_m1.jpg"+ },+ {+ "frame_id": 42,+ "timestamp": "2026-09-18T06:33:57.573050+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NASNETWORKRETURN NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713237573_m1.jpg"+ },+ {+ "frame_id": 41,+ "timestamp": "2026-09-18T06:33:27.094288+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NASNETWORKRETURN NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713207094_m1.jpg"+ },+ {+ "frame_id": 39,+ "timestamp": "2026-09-18T06:32:54.652491+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NASNETWORKRETURN NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713174652_m1.jpg"+ },+ {+ "frame_id": 38,+ "timestamp": "2026-09-18T06:31:37.936301+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NASNETWORKRETURN NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713097936_m1.jpg"+ },+ {+ "frame_id": 37,+ "timestamp": "2026-09-18T06:31:19.019688+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713079019_m1.jpg"+ },+ {+ "frame_id": 36,+ "timestamp": "2026-09-18T06:30:48.577376+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713048577_m1.jpg"+ },+ {+ "frame_id": 35,+ "timestamp": "2026-09-18T06:30:18.124923+00:00",+ "app_name": "TextEdit",+ "text": "NASPOSTMIGRATIONCONFIRMED 20260918 NAS AFTER MIGRATION CEDE 20260918\n\nScreenpipe is recording new activity after SQLite to Parquet migration.\nThis TextEdit window and its JPEG must remain in Timeline after app restart.\nThe data directory is on an SMB network share.\nEdited\n\u2014\nNAS-AFTER-MIGRATION-CEDE-20260918.txt",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/data/2026-09-18/1789713018124_m1.jpg"+ }+ ]+ },+ "retained-audio": {+ "pagination": {+ "limit": 10,+ "offset": 0,+ "total": 1+ },+ "data": [+ {+ "chunk_id": 2,+ "timestamp": "2026-09-18T06:27:00+00:00",+ "text": "audio device and stored on the network drive. The history remains searchable after restarting the application. Network storage validation. The recording continues after the migration. Both the old screen history and this new audio should still be available when the application starts again.",+ "file_path": "/Volumes/ScreenpipeApp/Mac Mini/data/System Audio (output)_2026-09-18_06-27-00.mp4"+ }+ ]+ }+}diff --git a/docs/testing/nas-storage/native-restart-status.json b/docs/testing/nas-storage/native-restart-status.jsonnew file mode 100644index 000000000..6a34c82ea--- /dev/null+++ b/docs/testing/nas-storage/native-restart-status.json@@ -0,0 +1,42 @@+[+ {+ "bytes_saved": null,+ "total_records": null,+ "elapsed_seconds": 0,+ "root": null,+ "recovering": false,+ "completed_records": null,+ "message": "",+ "busy": false,+ "available_bytes": null,+ "error": null,+ "completed": false+ },+ {+ "can_cancel": false,+ "message": "",+ "generation": "d9ae4f35-2564-4f49-a825-0a7fdeb20bfc",+ "can_delete_source": false,+ "busy": false,+ "error": null,+ "completed": true,+ "using_new_storage": true,+ "in_place": true,+ "pending": false,+ "app_session_id": "46996774-24c4-4ae9-9d89-448f43eb5850",+ "source_bytes": 0,+ "blocked_reason": null,+ "root": "/Volumes/ScreenpipeApp/Mac Mini",+ "available_bytes": 31886065664,+ "bytes_saved": 0,+ "can_migrate": false,+ "migrated_bytes": 6944768+ },+ {+ "phase": "ready",+ "message": null,+ "sinceEpochSecs": 1789713173,+ "error": null,+ "cpuCompatMode": false+ }+]diff --git a/docs/testing/nas-storage/native-timeline-state.json b/docs/testing/nas-storage/native-timeline-state.jsonnew file mode 100644index 000000000..c6a5e0b26--- /dev/null+++ b/docs/testing/nas-storage/native-timeline-state.json@@ -0,0 +1,73 @@+{+ "activeResultTimestamp": "2026-09-18T06:36:18.704Z",+ "highlightCount": 1,+ "loadedFrameIds": [+ "55",+ "54",+ "53",+ "52",+ "51",+ "50",+ "49",+ "48",+ "47",+ "46",+ "45",+ "44",+ "43",+ "42",+ "41",+ "40",+ "39",+ "38",+ "37",+ "36",+ "35",+ "34",+ "33",+ "32",+ "31",+ "30",+ "29",+ "28",+ "27",+ "26",+ "25",+ "24",+ "23",+ "22",+ "21",+ "20",+ "19",+ "18",+ "17",+ "16",+ "15",+ "14",+ "13",+ "12",+ "11",+ "10",+ "9",+ "8",+ "7",+ "6",+ "5",+ "4",+ "3",+ "2",+ "1"+ ],+ "attached": true,+ "activeResultIndex": 0,+ "queued": false,+ "activeResultFrameId": "47",+ "loadedImageFrameId": "47",+ "searchQuery": "NASNETWORKRETURN",+ "currentFrameId": "47",+ "displayedFrameId": "47",+ "currentTimestamp": "2026-09-18T06:36:18.704Z",+ "searchFrameIds": [+ "47"+ ]+}diff --git a/docs/testing/nas-storage/transport-stall.txt b/docs/testing/nas-storage/transport-stall.txtnew file mode 100644index 000000000..28c29573a--- /dev/null+++ b/docs/testing/nas-storage/transport-stall.txt@@ -0,0 +1,2 @@+Fri Sep 18 06:31:17 UTC 2026+Fri Sep 18 06:31:27 UTC 2026diff --git a/docs/testing/nas-storage/validation.md b/docs/testing/nas-storage/validation.mdnew file mode 100644index 000000000..d3a28b255--- /dev/null+++ b/docs/testing/nas-storage/validation.md@@ -0,0 +1,238 @@+# macOS NAS migration and recording validation++<!-- doc-covers: crates/screenpipe-fs, crates/screenpipe-db/src/storage, crates/screenpipe-db/src/db/setup.rs, crates/screenpipe-sqlite-coordinator/src/verification.rs, crates/screenpipe-screen/src/snapshot_writer.rs, crates/screenpipe-audio/src/audio_manager/reconciliation.rs, apps/screenpipe-app-tauri/components/storage-migration-prompt.tsx -->+<!-- doc-verified: c87fa3e65dc39ca93a65dc7865fe2266c886eab7 -->++Run: `nas-cede-20260918`, 2026-09-18 UTC. Baseline `c88ffa4b2d`;+product candidate `c87fa3e65d` (consumer 2.7.51).++The supplied 2.7.42 feedback showed a custom `/Volumes/...` data directory and+startup stuck in database migration. It did not include the native syscall+failure. The failures below were reproduced independently on a real SMB mount;+no customer database, account, or recordings were used or changed.++## Reproduction and cause++A fresh SIP-disabled macOS 26.6.2 (25G83) Tart guest mounted a Samba 4.19.5+SMB3 share from a disposable Ubuntu ARM64 VM. Both ran on the same Orchard Mac+worker. Guest isolation required an owned TCP relay on the VM gateway; this+was real macOS `smbfs`, not a mocked filesystem. The small storage test share+was a marked 512 MiB ext4 loop volume (487 MiB reported by SMB). Desktop tests+used a separate 35 GiB share with over 30 GiB free and a 110 GiB Mac guest+with over 70 GiB free locally.++[Filesystem probe](filesystem-probe.txt) and [baseline failure](baseline-smb.txt):++```text+F_FULLFSYNC -> ENOTSUP (45); fsync(file) and fsync(directory) -> success+F_PUNCHHOLE -> ENOTSUP (45)+baseline -> failed to arm SQLite verification incident:+ Operation not supported (os error 45)+```++Three independent incompatibilities had to be resolved:++1. Rust 1.94 uses `F_FULLFSYNC` for both `File::sync_all` and `sync_data` on+ Darwin. The NAS rejects this device-cache operation. The shared durability+ helper falls back to `fsync` only for Darwin ENOTSUP and propagates a failed+ fallback and all other I/O errors. Capture JPEGs, Parquet files, migration+ descriptors, verification markers, source identity, pending audio, settings,+ and connection configuration use this helper.+2. SMB rejects hole punching. Reclamation is now optional while the existing+ batch reserve and bounded in-place conversion remain mandatory. SQLite free+ pages stay available for reuse. There is no second database copy or VACUUM.+3. Forced `unix-excl` POSIX byte locks fail with `SQLITE_IOERR_LOCK` (3850).+ The default macOS VFS selects filesystem-aware locks. Its NAS path cannot+ reopen a shared-memory WAL (`SQLITE_CANTOPEN`, 14), so migration and runtime+ both use DELETE journaling and FULL synchronization on network volumes.+ Local volumes retain their existing VFS and WAL policy.++Legacy NAS WAL verification uses a SQL read-only barrier, an exclusive private+WAL index, and `NO_CKPT_ON_CLOSE`. The test byte-compares the database and hot,+committed WAL before/after verification, then proves normal recovery can adopt+those rows in rollback mode. Operational errors remain availability failures,+not evidence of corruption. The existing manager ownership and single writer+remain in place; this does not enable sharing one live database between Macs.++## Changed behavior++| Check | Result and evidence |+| --- | --- |+| Original unsupported operation | FAIL on baseline; preserved [failure log](baseline-smb.txt) |+| In-place conversion on non-sparse SMB | PASS: 48 historical frames and 48 elements; completion receipt present; migration journal removed |+| Recording after conversion and reopen | PASS: new frame, element and transcript sealed, searchable, exact payloads retained, `integrity_check=ok`; [log](migration-smb.txt) |+| Legacy committed WAL after process death | PASS: verifier leaves DB/WAL bytes unchanged, pending verification recovers, old/new rows survive reopen; [log](legacy-wal-smb.txt) |+| Migration/sealing process crashes | PASS: all 12 existing crash boundaries exercised on SMB; acknowledged rows survive resumed conversion; [log](crash-recovery-smb.txt) |+| Connection settings on SMB | PASS: create, replace and delete through the existing connection store |+| Ordinary app migration, capture and restart | Desktop acceptance recorded below |++The non-sparse test deliberately uses zero disk reserve and 1 MiB batches on+its disposable <=512 MiB share. Desktop acceptance uses production migration+budgets with no disk-pressure override. On the small share, source allocation+was 48,115,712 bytes, resulting index allocation 48,881,664 bytes and Parquet+154,176 bytes. This filesystem did **not** return SQLite free pages to the OS.+The prompt now explains reuse and reports measured savings without promising+that every NAS database file shrinks.++## Local checks++All commands run from the repository root unless an app-directory command is+shown. Hardware-specific NAS tests stay ignored in ordinary CI.++```sh+cargo test -p screenpipe-fs -p screenpipe-sqlite-coordinator --lib+# 3 durability + 27 coordinator tests passed+cargo test -p screenpipe-db --features storage-fault-injection --lib storage:: -- --nocapture+# 25 passed, 2 hardware-specific ignored+cargo test -p screenpipe-db --features storage-fault-injection \+ --test in_place_migration --test hybrid_storage --test bulk_storage \+ --test storage_snapshots --test sqlite_architecture_invariants_test \+ --test db_config_test --test multi_pool_wal_parity_test -- --nocapture+# 52 passed, 5 hardware-specific ignored+cargo test -p screenpipe-connect --lib mcp_servers::tests:: -- --nocapture+# 25 passed+cd apps/screenpipe-app-tauri+NODE_OPTIONS=--no-experimental-webstorage bun run test:vitest components/storage-migration-prompt.test.tsx+# 17 passed; Node 26 Web Storage otherwise masks jsdom localStorage+bun run build:tauri:e2e+# Packaged consumer debug-dev E2E build through the native queue and sccache+bun run coverage:all:check+# E2E, core and unified coverage reports passed+cd ../..+bun scripts/check-doc-freshness.ts --check+# Passed: no undeclared specs+git diff --check+# Passed+```++The E2E queue command was temporarily changed from `--no-bundle` to+`--bundles app` solely to package this test app. That harness edit and generated+E2E schemas are excluded from the PR. The source artifact was Apple Development+signed, verified, checksummed, and stored in a private Azure Blob container with+server-side encryption. Only the disposable guest copy was re-signed with+`tcc-grant --adhoc-sign`; all four capture permissions plus Full Disk Access+were granted. Feature-only onboarding and account fixtures bypass authentication;+they do not prove login, billing, cloud AI, or hosted enterprise ingest.++A paired APFS flush microbenchmark (ten alternating rounds, 100 4 KiB overwrites+per round) measured median 4.8225 ms before and 4.8559 ms after (+0.7%), with+substantially overlapping samples. [Raw samples](local-flush-benchmark.txt) and+[benchmark source](flush-benchmark.rs) are included. This is the changed flush+primitive, not a claim about end-to-end NAS throughput. Local successful flushes+add no filesystem lookup, allocation, or syscall. NAS latency depends on the+network and server; successful flush durability depends on the server.++## Repeating the NAS tests++Use only an owned disposable share with `.screenpipe-disposable-volume` at its+root. Compile the normal macOS test executables with the commands above and+transfer them to the Mac guest. The fixture code and [filesystem probe](filesystem-probe.py)+are included. The baseline test is named `unsupported_volume_fails_before_conversion`.++```sh+SCREENPIPE_UNSUPPORTED_VOLUME=/Volumes/Screenpipe ./in_place_migration \+ non_sparse_volume_migrates_and_keeps_recording_after_restart --ignored --exact --nocapture+SCREENPIPE_UNSUPPORTED_VOLUME=/Volumes/Screenpipe ./in_place_migration \+ network_wal_verification_and_reopen_preserve_committed_rows --ignored --exact --nocapture+TMPDIR=/Volumes/ScreenpipeApp/tests ./hybrid_storage \+ migration_crashes_resume_without_losing_acknowledged_records --exact --nocapture+```++The hybrid crash test launches `screenpipe-storage` from its compile-time path;+install the matching CLI there inside the disposable guest. The desktop legacy+history fixture uses the production database writer; its eight synthetic rows+and placeholder JPEG are distinct from the subsequent real TextEdit captures.++## Desktop acceptance++The packaged app used product commit `c87fa3e65dc39ca93a65dc7865fe2266c886eab7`.+This is the NAS acceptance build. Subsequent CI repairs update test fixtures,+the Intel smoke build's sidecar selection, and Windows recovery-owner detection;+they do not change the NAS filesystem or migration implementation. Their check+results are recorded in PR #7090. The private source bundle SHA-256 is+`5da727e13a7d3a6bfbfb898cccce3e023bb208f6bb2df3d9118d06052b1f5b16`.+The 53-second migration video SHA-256 is+`5dddb3f77dfe7b49a18f59381327ab5e2beca436022f1b4e290ce2e329372d2b`.+Both objects are in a private, server-encrypted Azure container. The PR includes+a read-only video link expiring September 24, 2026; the app bundle stays private.++The app's `SCREENPIPE_DATA_DIR` was `/Volumes/ScreenpipeApp/Mac Mini`. Clicking+**Start now** migrated 332 records, including 23 frames, in 19 seconds. This+included eight synthetic legacy frames and real TextEdit captures. The+[completion receipt](native-completion-receipt.json) was present, the in-place+journal was absent, and [native status](native-migration-status.json) reported+`completed=true`, `using_new_storage=true`, `pending=false`, with no error.+Allocated storage grew from 6,037,504 to 6,944,768 bytes; the app correctly+reported zero bytes saved on this non-sparse share.++The PR embeds the before/after screenshots (`migration-before.png` and+`migration-complete.png`) and links the migration recording. Media is hosted+externally, as required by the PR template.++After conversion, actual TextEdit accessibility text and JPEGs continued writing+to the NAS. A 10-second relay pause from 06:31:17 to 06:31:27 UTC exercised a+bounded transport stall. [Post-stall counters](native-after-stall-health.json)+reported 12 captured frames, 12 database writes, zero drops, zero silent loss,+and no degraded writer. Frame 37, captured during the pause, remained readable+after restart. This was a transport stall, not an unmount or server power-loss+test.++The normal native `stop_screenpipe` command drained recording and closed the+pools; the owned app process was then terminated and relaunched with the same+data directory. Offline `screenpipe-storage verify` returned `{"verified":true}`+both before this restart and after final shutdown. The restarted app retained+the same storage generation and reached `ready` without rerunning migration:+[restart status](native-restart-status.json).++[Authenticated API results after restart](native-restart-search.json) contain+all eight legacy rows, pre-restart frames 35–38, new post-restart frames, and the+NAS audio transcript. The real Search card opened frame 47 in the native+Timeline; [native readback](native-timeline-state.json) confirms current,+displayed, and loaded-image frame IDs all equal 47. The screenshot shows the+captured `NASNETWORKRETURN` and `NASPOSTMIGRATIONCONFIRMED` markers:++The PR embeds `timeline-new-recording.png`, the actual native Timeline display.++The final [restart health](native-restart-health.json) reports 18 captured and+18 written frames, no dropped frames or writer failures, and active UI-event+recording. Its overall health is **degraded for audio**, not an all-green pass.++### Audio result and testing limits++Through the real settings UI, audio recording was enabled and Whisper Tiny+selected. Synthetic speech played inside the VM through CoreAudio. Raw system+audio was saved on the NAS; a 30.01-second file decoded successfully, with peak+volume -3.3 dB. The automatic VAD rejected the synthetic input, so automatic+transcription was not proven. There were no audio processing or storage errors.++The normal `/audio/retranscribe` API with Whisper Tiny processed five recorded+chunks and transcribed the audible chunk successfully:+[retranscription result](audio-retranscription.json). Its transcript and NAS file+remained searchable after restart. This proves actual audio capture, NAS file+read/decode, local transcription, database persistence, and search/reopen; it+does not claim automatic VAD acceptance or cloud transcription.++The first fixture mount detached when SMB multichannel attempted an unreachable+guest-to-guest route around the required relay. The source database remained+intact. Disabling multichannel on the disposable client and server corrected+that test-network configuration before the final acceptance run. No product+code was changed for that routing problem. Search was refreshed after its+normal indexing exclusion window; the backend and native Timeline were tested+with actual persisted frames.++## Cleanup++The app was stopped and the final database verified before teardown. Orchard+deleted exactly `nas-cede-mac-20260918` and `nas-cede-smb-20260918`. Orchard and+the worker's Tart inventory confirmed both gone. The owned TCP relay process+and script were removed, port 1445 had no listener, and the run lock was+released. The original `aws-orchard` context was unchanged. VM deletion removed+the synthetic account fixture, local API key, imported test app, data volumes,+and ephemeral SSH authorization. Base images and unrelated stopped VMs were+left intact. No host application was launched or stopped.++This validates macOS SMB against the described Samba server. The customer's+macOS 27 installation and NAS firmware were not directly available. NFS,+Windows NAS mounts, server power loss, and concurrent multi-host ownership were+not tested. Existing APFS behavior and typed I/O error propagation were tested+locally. No app release or updater pointer was published.diff --git a/packages/sdk/Cargo.lock b/packages/sdk/Cargo.lockindex 7041d389f..99211aea1 100644--- a/packages/sdk/Cargo.lock+++ b/packages/sdk/Cargo.lock@@ -6920,6 +6920,7 @@ dependencies = ["russh-sftp","rustls-native-certs","screenpipe-core",+ "screenpipe-fs","screenpipe-secrets","screenpipe-vault","serde",@@ -6963,6 +6964,7 @@ dependencies = ["reqwest","screenpipe-cpu-features","screenpipe-events",+ "screenpipe-fs","serde","serde_json","serde_yaml",@@ -6999,6 +7001,7 @@ dependencies = ["regex","rusqlite","screenpipe-config",+ "screenpipe-fs","screenpipe-resource","screenpipe-semantic","screenpipe-sqlite-coordinator",@@ -7033,6 +7036,13 @@ dependencies = ["tracing",]+[[package]]+name = "screenpipe-fs"+version = "0.4.50"+dependencies = [+ "libc",+]+[[package]]name = "screenpipe-recorder"version = "0.4.3"@@ -7091,6 +7101,7 @@ dependencies = ["screenpipe-connect","screenpipe-core","screenpipe-db",+ "screenpipe-fs","serde","serde_json","tokio",@@ -7148,6 +7159,7 @@ version = "0.4.50"dependencies = ["fs2","libsqlite3-sys",+ "screenpipe-fs","serde","serde_json","sqlx",diff --git a/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock b/packages/sdk/examples/tauri-app/src-tauri/Cargo.lockindex a453cd767..b927ba958 100644--- a/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock+++ b/packages/sdk/examples/tauri-app/src-tauri/Cargo.lock@@ -7046,6 +7046,7 @@ dependencies = ["russh-sftp","rustls-native-certs","screenpipe-core",+ "screenpipe-fs","screenpipe-secrets","screenpipe-vault","serde",@@ -7089,6 +7090,7 @@ dependencies = ["reqwest","screenpipe-cpu-features","screenpipe-events",+ "screenpipe-fs","serde","serde_json","serde_yaml",@@ -7125,6 +7127,7 @@ dependencies = ["regex","rusqlite","screenpipe-config",+ "screenpipe-fs","screenpipe-resource","screenpipe-semantic","screenpipe-sqlite-coordinator",@@ -7159,6 +7162,13 @@ dependencies = ["tracing",]+[[package]]+name = "screenpipe-fs"+version = "0.4.50"+dependencies = [+ "libc",+]+[[package]]name = "screenpipe-recorder"version = "0.4.3"@@ -7215,6 +7225,7 @@ dependencies = ["screenpipe-connect","screenpipe-core","screenpipe-db",+ "screenpipe-fs","serde","serde_json","tokio",@@ -7261,6 +7272,7 @@ version = "0.4.50"dependencies = ["fs2","libsqlite3-sys",+ "screenpipe-fs","serde","serde_json","sqlx",