Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(app): stop repeated filesystem CPU storms (#6767)

Louis Beaumontd0d19451b32c
diff --git a/apps/screenpipe-app-tauri/components/__tests__/chat-sidebar-focus.test.tsx b/apps/screenpipe-app-tauri/components/__tests__/chat-sidebar-focus.test.tsx
index 5211e2b72..d2dfdef5b 100644
--- a/apps/screenpipe-app-tauri/components/__tests__/chat-sidebar-focus.test.tsx
+++ b/apps/screenpipe-app-tauri/components/__tests__/chat-sidebar-focus.test.tsx
@@ -5,6 +5,7 @@
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
+ CHAT_SIDEBAR_HYDRATION_OPTIONS,
filterRecentsBySource,
handleMenuShortcut,
hiddenRecentSourcesFromStoredValue,
@@ -15,6 +16,7 @@ import {
visibleRecentSourceOptions,
} from "@/components/chat-sidebar";
import { TooltipProvider } from "@/components/ui/tooltip";
+import { CHAT_HISTORY_INITIAL_LIMIT } from "@/lib/chat-storage";
import type { SessionRecord } from "@/lib/stores/chat-store";
const baseSession: SessionRecord = {
@@ -35,6 +37,15 @@ beforeAll(() => {
globalThis.PointerEvent ||= MouseEvent as unknown as typeof PointerEvent;
});
+describe("chat sidebar hydration", () => {
+ it("bounds disk hydration to the recent chat window", () => {
+ expect(CHAT_SIDEBAR_HYDRATION_OPTIONS).toEqual({
+ limit: CHAT_HISTORY_INITIAL_LIMIT,
+ includeHidden: true,
+ });
+ });
+});
+
function renderRow(
isCurrent: boolean,
status: SessionRecord["status"] = "idle",
diff --git a/apps/screenpipe-app-tauri/components/chat-sidebar.tsx b/apps/screenpipe-app-tauri/components/chat-sidebar.tsx
index 6fa58da07..5c52ea0e2 100644
--- a/apps/screenpipe-app-tauri/components/chat-sidebar.tsx
+++ b/apps/screenpipe-app-tauri/components/chat-sidebar.tsx
@@ -66,6 +66,7 @@ import {
type SessionRecord,
} from "@/lib/stores/chat-store";
import {
+ CHAT_HISTORY_INITIAL_LIMIT,
conversationMetaFromJson,
deleteConversationFile,
listConversations,
@@ -172,6 +173,10 @@ import {
/** Max top-level rows shown in recents. Pipes use the authoritative inventory. */
const SIDEBAR_CAP = 8;
+export const CHAT_SIDEBAR_HYDRATION_OPTIONS = {
+ limit: CHAT_HISTORY_INITIAL_LIMIT,
+ includeHidden: true,
+} as const;
const PIPE_RUNS_PER_GROUP = 10;
const DELETED_PIPE_EXECUTIONS_KEY = "screenpipe:deleted-pipe-executions";
const RECENTS_SOURCE_FILTER_KEY = "screenpipe:recents-hidden-sources";
@@ -680,7 +685,9 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {
let cancelled = false;
let controller: ExternalChatSyncController | null = null;
const hydrate = async () => {
- const metas = await listConversations({ includeHidden: true });
+ // Each item crosses the Tauri filesystem boundary. Keep this bounded:
+ // large imported histories can contain tens of thousands of chat files.
+ const metas = await listConversations(CHAT_SIDEBAR_HYDRATION_OPTIONS);
if (!cancelled) actions.hydrateFromDisk(metas.map(sessionRecordFromMeta));
};
const start = async () => {
@@ -699,9 +706,11 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {
void start();
const onFocus = () => {
if (!controller) return;
- void controller.syncNow().then(hydrate).catch((error) => {
- console.warn("[chat-sidebar] external chat reconciliation failed", error);
- });
+ void controller.syncNow()
+ .then((reconciled) => reconciled ? hydrate() : undefined)
+ .catch((error) => {
+ console.warn("[chat-sidebar] external chat reconciliation failed", error);
+ });
};
window.addEventListener("focus", onFocus);
return () => {
diff --git a/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.test.ts b/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.test.ts
index aa1e2e599..34029a966 100644
--- a/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.test.ts
+++ b/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.test.ts
@@ -170,7 +170,8 @@ describe("external chat live sync", () => {
paths: [`${codexRoot}/2026/08/26/ignored.jsonl`],
attrs: {},
});
- await controller.syncNow();
+ expect(await controller.syncNow()).toBe(false);
+ expect(mocks.scanExternalChatHistory).toHaveBeenCalledTimes(1);
expect(mocks.candidateForPath).toHaveBeenCalledTimes(1);
controller.stop();
diff --git a/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.ts b/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.ts
index 84c432759..63fb4ca69 100644
--- a/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.ts
+++ b/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.ts
@@ -30,7 +30,7 @@ interface WatchTarget {
}
export interface ExternalChatSyncController {
- syncNow(force?: boolean): Promise<void>;
+ syncNow(force?: boolean): Promise<boolean>;
stop(): void;
}
@@ -102,7 +102,7 @@ export async function startExternalChatSync(
const home = await resolveExternalChatHome(options.home);
if (!home) {
return {
- syncNow: async () => {},
+ syncNow: async () => false,
stop: () => {},
};
}
@@ -160,20 +160,24 @@ export async function startExternalChatSync(
};
const controller: ExternalChatSyncController = {
- syncNow: (force = false) => enqueue(async () => {
- await ensureWatchers();
- const now = Date.now();
- if (!force && now - lastFullSyncAt < FULL_RECONCILIATION_INTERVAL_MS) return;
- lastFullSyncAt = now;
- try {
- const scan = await scanExternalChatHistory({ home });
- await importExternalChatHistory(candidatesFromScan(scan), {
- skipUnchanged: true,
- });
- } catch (error) {
- console.warn("[chat-sync] failed to reconcile external chat history", error);
- }
- }),
+ syncNow: (force = false) => {
+ let reconciled = false;
+ return enqueue(async () => {
+ await ensureWatchers();
+ const now = Date.now();
+ if (!force && now - lastFullSyncAt < FULL_RECONCILIATION_INTERVAL_MS) return;
+ lastFullSyncAt = now;
+ try {
+ const scan = await scanExternalChatHistory({ home });
+ await importExternalChatHistory(candidatesFromScan(scan), {
+ skipUnchanged: true,
+ });
+ reconciled = true;
+ } catch (error) {
+ console.warn("[chat-sync] failed to reconcile external chat history", error);
+ }
+ }).then(() => reconciled);
+ },
stop: () => {
stopped = true;
for (const unwatch of unwatchByRoot.values()) unwatch();
diff --git a/crates/screenpipe-core/src/agents/pi.rs b/crates/screenpipe-core/src/agents/pi.rs
index c063d5930..771a4736c 100644
--- a/crates/screenpipe-core/src/agents/pi.rs
+++ b/crates/screenpipe-core/src/agents/pi.rs
@@ -11,11 +11,54 @@ use super::{install_spawned_pid, AgentExecutor, AgentOutput, ExecutionHandle};
use anyhow::{anyhow, Result};
use arc_swap::ArcSwap;
use serde_json::json;
+use sha2::{Digest, Sha256};
use std::ffi::{OsStr, OsString};
+use std::io::Read;
use std::path::{Path, PathBuf};
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
use tracing::{debug, error, info, warn};
+static USER_SKILL_SYNC_LOCK: Mutex<()> = Mutex::new(());
+
+fn user_skill_fingerprint(root: &Path) -> std::io::Result<String> {
+ fn hash_dir(root: &Path, dir: &Path, hasher: &mut Sha256) -> std::io::Result<()> {
+ let mut entries = std::fs::read_dir(dir)?.collect::<std::io::Result<Vec<_>>>()?;
+ entries.sort_by_key(|entry| entry.file_name());
+
+ for entry in entries {
+ let file_type = entry.file_type()?;
+ if !file_type.is_dir() && !file_type.is_file() {
+ continue;
+ }
+
+ let path = entry.path();
+ let relative = path.strip_prefix(root).unwrap_or(&path).to_string_lossy();
+ hasher.update(if file_type.is_dir() { b"d" } else { b"f" });
+ hasher.update((relative.len() as u64).to_le_bytes());
+ hasher.update(relative.as_bytes());
+
+ if file_type.is_dir() {
+ hash_dir(root, &path, hasher)?;
+ } else {
+ let mut file = std::fs::File::open(path)?;
+ let mut buffer = [0_u8; 16 * 1024];
+ loop {
+ let read = file.read(&mut buffer)?;
+ if read == 0 {
+ break;
+ }
+ hasher.update(&buffer[..read]);
+ }
+ }
+ }
+ Ok(())
+ }
+
+ let mut hasher = Sha256::new();
+ hash_dir(root, root, &mut hasher)?;
+ Ok(format!("{:x}", hasher.finalize()))
+}
+
pub const PI_PACKAGE: &str = "@earendil-works/[email protected]";
pub const PI_AI_PACKAGE: &str = "@earendil-works/[email protected]";
pub const PI_NAMESPACE_DIR: &str = "@earendil-works";
@@ -868,7 +911,8 @@ impl PiExecutor {
///
/// Idempotent + self-cleaning: each mirrored skill is stamped with
/// [`Self::USER_SKILL_MARKER`]; on every call we refresh the contents of
- /// skills still in the store and remove previously-mirrored skills that
+ /// skills changed in the store, skip managed copies whose recorded source
+ /// fingerprint still matches, and remove previously-mirrored skills that
/// have left it. Baseline + hand-authored skills (no marker) are never
/// touched. Best-effort: a single malformed skill is logged and skipped so
/// it can never break a session.
@@ -880,6 +924,9 @@ impl PiExecutor {
/// Implementation of [`Self::sync_user_skills`] with the store path passed
/// in, so it can be unit-tested without touching the real data dir.
fn sync_user_skills_from(store: &Path, project_dir: &Path) -> Result<()> {
+ let _sync_guard = USER_SKILL_SYNC_LOCK
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
let dest_root = project_dir.join(".pi").join("skills");
// Copy/refresh every store skill (a folder containing SKILL.md).
@@ -900,22 +947,27 @@ impl PiExecutor {
if Self::BASELINE_SKILL_NAMES.contains(&key.as_str()) {
continue;
}
+ store_keys.insert(key.clone());
let dest = dest_root.join(&key);
let copy = (|| -> std::io::Result<()> {
+ let fingerprint = user_skill_fingerprint(&src)?;
+ let marker = format!(
+ "mirrored from <data>/skills by screenpipe\nfingerprint={fingerprint}\n"
+ );
+ if std::fs::read_to_string(dest.join(Self::USER_SKILL_MARKER))
+ .is_ok_and(|existing| existing == marker)
+ {
+ return Ok(());
+ }
if dest.exists() {
std::fs::remove_dir_all(&dest)?;
}
crate::paths::copy_dir_all(&src, &dest)?;
- std::fs::write(
- dest.join(Self::USER_SKILL_MARKER),
- b"mirrored from <data>/skills by screenpipe\n",
- )?;
+ std::fs::write(dest.join(Self::USER_SKILL_MARKER), marker)?;
Ok(())
})();
match copy {
- Ok(()) => {
- store_keys.insert(key);
- }
+ Ok(()) => {}
Err(e) => warn!("failed to mirror user skill {:?}: {}", src, e),
}
}
@@ -4688,6 +4740,26 @@ mod tests {
.join(PiExecutor::USER_SKILL_MARKER)
.exists());
+ // An unchanged source preserves the managed copy instead of deleting
+ // and recursively copying the full skill tree again.
+ std::fs::write(skills.join("foo").join("copy-sentinel"), "preserved").unwrap();
+ PiExecutor::sync_user_skills_from(&store, &project).unwrap();
+ assert!(skills.join("foo").join("copy-sentinel").exists());
+
+ // A source content change invalidates the marker and refreshes the
+ // managed copy, removing anything that is no longer in the source.
+ std::fs::write(
+ store.join("foo").join("SKILL.md"),
+ "---\nname: foo\n---\nupdated",
+ )
+ .unwrap();
+ PiExecutor::sync_user_skills_from(&store, &project).unwrap();
+ assert_eq!(
+ std::fs::read_to_string(skills.join("foo").join("SKILL.md")).unwrap(),
+ "---\nname: foo\n---\nupdated"
+ );
+ assert!(!skills.join("foo").join("copy-sentinel").exists());
+
// Remove from store, sync again → our mirror is gone, baseline stays.
std::fs::remove_dir_all(store.join("foo")).unwrap();
PiExecutor::sync_user_skills_from(&store, &project).unwrap();