YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(app): stop repeated filesystem CPU storms (#6767)
Louis Beaumontd0d19451b32c
diff --git a/apps/screenpipe-app-tauri/components/__tests__/chat-sidebar-focus.test.tsx b/apps/screenpipe-app-tauri/components/__tests__/chat-sidebar-focus.test.tsxindex 5211e2b72..d2dfdef5b 100644--- a/apps/screenpipe-app-tauri/components/__tests__/chat-sidebar-focus.test.tsx+++ b/apps/screenpipe-app-tauri/components/__tests__/chat-sidebar-focus.test.tsx@@ -5,6 +5,7 @@import { cleanup, fireEvent, render, screen } from "@testing-library/react";import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";import {+ CHAT_SIDEBAR_HYDRATION_OPTIONS,filterRecentsBySource,handleMenuShortcut,hiddenRecentSourcesFromStoredValue,@@ -15,6 +16,7 @@ import {visibleRecentSourceOptions,} from "@/components/chat-sidebar";import { TooltipProvider } from "@/components/ui/tooltip";+import { CHAT_HISTORY_INITIAL_LIMIT } from "@/lib/chat-storage";import type { SessionRecord } from "@/lib/stores/chat-store";const baseSession: SessionRecord = {@@ -35,6 +37,15 @@ beforeAll(() => {globalThis.PointerEvent ||= MouseEvent as unknown as typeof PointerEvent;});+describe("chat sidebar hydration", () => {+ it("bounds disk hydration to the recent chat window", () => {+ expect(CHAT_SIDEBAR_HYDRATION_OPTIONS).toEqual({+ limit: CHAT_HISTORY_INITIAL_LIMIT,+ includeHidden: true,+ });+ });+});+function renderRow(isCurrent: boolean,status: SessionRecord["status"] = "idle",diff --git a/apps/screenpipe-app-tauri/components/chat-sidebar.tsx b/apps/screenpipe-app-tauri/components/chat-sidebar.tsxindex 6fa58da07..5c52ea0e2 100644--- a/apps/screenpipe-app-tauri/components/chat-sidebar.tsx+++ b/apps/screenpipe-app-tauri/components/chat-sidebar.tsx@@ -66,6 +66,7 @@ import {type SessionRecord,} from "@/lib/stores/chat-store";import {+ CHAT_HISTORY_INITIAL_LIMIT,conversationMetaFromJson,deleteConversationFile,listConversations,@@ -172,6 +173,10 @@ import {/** Max top-level rows shown in recents. Pipes use the authoritative inventory. */const SIDEBAR_CAP = 8;+export const CHAT_SIDEBAR_HYDRATION_OPTIONS = {+ limit: CHAT_HISTORY_INITIAL_LIMIT,+ includeHidden: true,+} as const;const PIPE_RUNS_PER_GROUP = 10;const DELETED_PIPE_EXECUTIONS_KEY = "screenpipe:deleted-pipe-executions";const RECENTS_SOURCE_FILTER_KEY = "screenpipe:recents-hidden-sources";@@ -680,7 +685,9 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {let cancelled = false;let controller: ExternalChatSyncController | null = null;const hydrate = async () => {- const metas = await listConversations({ includeHidden: true });+ // Each item crosses the Tauri filesystem boundary. Keep this bounded:+ // large imported histories can contain tens of thousands of chat files.+ const metas = await listConversations(CHAT_SIDEBAR_HYDRATION_OPTIONS);if (!cancelled) actions.hydrateFromDisk(metas.map(sessionRecordFromMeta));};const start = async () => {@@ -699,9 +706,11 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {void start();const onFocus = () => {if (!controller) return;- void controller.syncNow().then(hydrate).catch((error) => {- console.warn("[chat-sidebar] external chat reconciliation failed", error);- });+ void controller.syncNow()+ .then((reconciled) => reconciled ? hydrate() : undefined)+ .catch((error) => {+ console.warn("[chat-sidebar] external chat reconciliation failed", error);+ });};window.addEventListener("focus", onFocus);return () => {diff --git a/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.test.ts b/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.test.tsindex aa1e2e599..34029a966 100644--- a/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.test.ts+++ b/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.test.ts@@ -170,7 +170,8 @@ describe("external chat live sync", () => {paths: [`${codexRoot}/2026/08/26/ignored.jsonl`],attrs: {},});- await controller.syncNow();+ expect(await controller.syncNow()).toBe(false);+ expect(mocks.scanExternalChatHistory).toHaveBeenCalledTimes(1);expect(mocks.candidateForPath).toHaveBeenCalledTimes(1);controller.stop();diff --git a/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.ts b/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.tsindex 84c432759..63fb4ca69 100644--- a/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.ts+++ b/apps/screenpipe-app-tauri/lib/chat/external-chat-sync.ts@@ -30,7 +30,7 @@ interface WatchTarget {}export interface ExternalChatSyncController {- syncNow(force?: boolean): Promise<void>;+ syncNow(force?: boolean): Promise<boolean>;stop(): void;}@@ -102,7 +102,7 @@ export async function startExternalChatSync(const home = await resolveExternalChatHome(options.home);if (!home) {return {- syncNow: async () => {},+ syncNow: async () => false,stop: () => {},};}@@ -160,20 +160,24 @@ export async function startExternalChatSync(};const controller: ExternalChatSyncController = {- syncNow: (force = false) => enqueue(async () => {- await ensureWatchers();- const now = Date.now();- if (!force && now - lastFullSyncAt < FULL_RECONCILIATION_INTERVAL_MS) return;- lastFullSyncAt = now;- try {- const scan = await scanExternalChatHistory({ home });- await importExternalChatHistory(candidatesFromScan(scan), {- skipUnchanged: true,- });- } catch (error) {- console.warn("[chat-sync] failed to reconcile external chat history", error);- }- }),+ syncNow: (force = false) => {+ let reconciled = false;+ return enqueue(async () => {+ await ensureWatchers();+ const now = Date.now();+ if (!force && now - lastFullSyncAt < FULL_RECONCILIATION_INTERVAL_MS) return;+ lastFullSyncAt = now;+ try {+ const scan = await scanExternalChatHistory({ home });+ await importExternalChatHistory(candidatesFromScan(scan), {+ skipUnchanged: true,+ });+ reconciled = true;+ } catch (error) {+ console.warn("[chat-sync] failed to reconcile external chat history", error);+ }+ }).then(() => reconciled);+ },stop: () => {stopped = true;for (const unwatch of unwatchByRoot.values()) unwatch();diff --git a/crates/screenpipe-core/src/agents/pi.rs b/crates/screenpipe-core/src/agents/pi.rsindex c063d5930..771a4736c 100644--- a/crates/screenpipe-core/src/agents/pi.rs+++ b/crates/screenpipe-core/src/agents/pi.rs@@ -11,11 +11,54 @@ use super::{install_spawned_pid, AgentExecutor, AgentOutput, ExecutionHandle};use anyhow::{anyhow, Result};use arc_swap::ArcSwap;use serde_json::json;+use sha2::{Digest, Sha256};use std::ffi::{OsStr, OsString};+use std::io::Read;use std::path::{Path, PathBuf};-use std::sync::Arc;+use std::sync::{Arc, Mutex};use tracing::{debug, error, info, warn};+static USER_SKILL_SYNC_LOCK: Mutex<()> = Mutex::new(());++fn user_skill_fingerprint(root: &Path) -> std::io::Result<String> {+ fn hash_dir(root: &Path, dir: &Path, hasher: &mut Sha256) -> std::io::Result<()> {+ let mut entries = std::fs::read_dir(dir)?.collect::<std::io::Result<Vec<_>>>()?;+ entries.sort_by_key(|entry| entry.file_name());++ for entry in entries {+ let file_type = entry.file_type()?;+ if !file_type.is_dir() && !file_type.is_file() {+ continue;+ }++ let path = entry.path();+ let relative = path.strip_prefix(root).unwrap_or(&path).to_string_lossy();+ hasher.update(if file_type.is_dir() { b"d" } else { b"f" });+ hasher.update((relative.len() as u64).to_le_bytes());+ hasher.update(relative.as_bytes());++ if file_type.is_dir() {+ hash_dir(root, &path, hasher)?;+ } else {+ let mut file = std::fs::File::open(path)?;+ let mut buffer = [0_u8; 16 * 1024];+ loop {+ let read = file.read(&mut buffer)?;+ if read == 0 {+ break;+ }+ hasher.update(&buffer[..read]);+ }+ }+ }+ Ok(())+ }++ let mut hasher = Sha256::new();+ hash_dir(root, root, &mut hasher)?;+ Ok(format!("{:x}", hasher.finalize()))+}+pub const PI_PACKAGE: &str = "@earendil-works/[email protected]";pub const PI_AI_PACKAGE: &str = "@earendil-works/[email protected]";pub const PI_NAMESPACE_DIR: &str = "@earendil-works";@@ -868,7 +911,8 @@ impl PiExecutor {////// Idempotent + self-cleaning: each mirrored skill is stamped with/// [`Self::USER_SKILL_MARKER`]; on every call we refresh the contents of- /// skills still in the store and remove previously-mirrored skills that+ /// skills changed in the store, skip managed copies whose recorded source+ /// fingerprint still matches, and remove previously-mirrored skills that/// have left it. Baseline + hand-authored skills (no marker) are never/// touched. Best-effort: a single malformed skill is logged and skipped so/// it can never break a session.@@ -880,6 +924,9 @@ impl PiExecutor {/// Implementation of [`Self::sync_user_skills`] with the store path passed/// in, so it can be unit-tested without touching the real data dir.fn sync_user_skills_from(store: &Path, project_dir: &Path) -> Result<()> {+ let _sync_guard = USER_SKILL_SYNC_LOCK+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner());let dest_root = project_dir.join(".pi").join("skills");// Copy/refresh every store skill (a folder containing SKILL.md).@@ -900,22 +947,27 @@ impl PiExecutor {if Self::BASELINE_SKILL_NAMES.contains(&key.as_str()) {continue;}+ store_keys.insert(key.clone());let dest = dest_root.join(&key);let copy = (|| -> std::io::Result<()> {+ let fingerprint = user_skill_fingerprint(&src)?;+ let marker = format!(+ "mirrored from <data>/skills by screenpipe\nfingerprint={fingerprint}\n"+ );+ if std::fs::read_to_string(dest.join(Self::USER_SKILL_MARKER))+ .is_ok_and(|existing| existing == marker)+ {+ return Ok(());+ }if dest.exists() {std::fs::remove_dir_all(&dest)?;}crate::paths::copy_dir_all(&src, &dest)?;- std::fs::write(- dest.join(Self::USER_SKILL_MARKER),- b"mirrored from <data>/skills by screenpipe\n",- )?;+ std::fs::write(dest.join(Self::USER_SKILL_MARKER), marker)?;Ok(())})();match copy {- Ok(()) => {- store_keys.insert(key);- }+ Ok(()) => {}Err(e) => warn!("failed to mirror user skill {:?}: {}", src, e),}}@@ -4688,6 +4740,26 @@ mod tests {.join(PiExecutor::USER_SKILL_MARKER).exists());+ // An unchanged source preserves the managed copy instead of deleting+ // and recursively copying the full skill tree again.+ std::fs::write(skills.join("foo").join("copy-sentinel"), "preserved").unwrap();+ PiExecutor::sync_user_skills_from(&store, &project).unwrap();+ assert!(skills.join("foo").join("copy-sentinel").exists());++ // A source content change invalidates the marker and refreshes the+ // managed copy, removing anything that is no longer in the source.+ std::fs::write(+ store.join("foo").join("SKILL.md"),+ "---\nname: foo\n---\nupdated",+ )+ .unwrap();+ PiExecutor::sync_user_skills_from(&store, &project).unwrap();+ assert_eq!(+ std::fs::read_to_string(skills.join("foo").join("SKILL.md")).unwrap(),+ "---\nname: foo\n---\nupdated"+ );+ assert!(!skills.join("foo").join("copy-sentinel").exists());+// Remove from store, sync again → our mirror is gone, baseline stays.std::fs::remove_dir_all(store.join("foo")).unwrap();PiExecutor::sync_user_skills_from(&store, &project).unwrap();