YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
Fix pipe run history pagination
Louis Beaumont152be2ad0a43
diff --git a/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx b/apps/screenpipe-app-tauri/components/settings/pipes-section.tsxindex 9938c49b1..9b86961e6 100644--- a/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx+++ b/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx@@ -129,6 +129,18 @@ import { useDeviceMonitor } from "@/lib/hooks/use-device-monitor";import { Monitor, Wifi, WifiOff, ScanSearch } from "lucide-react";import { requestPipeStop } from "@/lib/pipe-stop";+const PIPE_EXECUTIONS_PAGE_LIMIT = 100;++function pipeExecutionsUrl(apiBase: string, pipeName: string, beforeId?: number) {+ const params = new URLSearchParams({+ limit: String(PIPE_EXECUTIONS_PAGE_LIMIT),+ });+ if (beforeId != null) {+ params.set("before_id", String(beforeId));+ }+ return `${apiBase}/pipes/${encodeURIComponent(pipeName)}/executions?${params.toString()}`;+}+const PIPE_CREATION_PROMPT = `create a screenpipe pipe that does the following.## what is screenpipe?@@ -1018,6 +1030,8 @@ export function PipesSection() {const [logs, setLogs] = useState<PipeRunLog[]>([]);const [executions, setExecutions] = useState<PipeExecution[]>([]);const [executionsLoading, setExecutionsLoading] = useState(false);+ const [hasMoreExecutions, setHasMoreExecutions] = useState(false);+ const [loadingMoreExecutions, setLoadingMoreExecutions] = useState(false);// Per-pipe recent executions (always fetched for all pipes)const [pipeExecutions, setPipeExecutions] = useState<Record<string, PipeExecution[]>>({});const [loading, setLoading] = useState(true);@@ -1640,9 +1654,11 @@ export function PipesSection() {const exp = expandedRef.current;if (exp) {try {- const execRes = await fetch(`${apiBase}/pipes/${exp}/executions?limit=20`);+ const execRes = await fetch(pipeExecutionsUrl(apiBase, exp));const execData = await execRes.json();- setExecutions(execData.data || []);+ const nextExecutions = execData.data || [];+ setExecutions(nextExecutions);+ setHasMoreExecutions(nextExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);const finishedKeys = (execData.data || []).filter((e: PipeExecution) => e.status !== "running").map((e: PipeExecution) => `${e.pipe_name}:${e.id}`);@@ -1688,18 +1704,47 @@ export function PipesSection() {const fetchExecutions = async (name: string) => {setExecutionsLoading(true);+ setHasMoreExecutions(false);try {- const res = await fetch(`${apiBase}/pipes/${name}/executions?limit=20`);+ const res = await fetch(pipeExecutionsUrl(apiBase, name));const data = await res.json();- setExecutions(data.data || []);+ const nextExecutions = data.data || [];+ setExecutions(nextExecutions);+ setHasMoreExecutions(nextExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);} catch (e) {// Executions endpoint may not exist on older servers — fall back silentlysetExecutions([]);+ setHasMoreExecutions(false);} finally {setExecutionsLoading(false);}};+ const loadMoreExecutions = async (name: string) => {+ if (loadingMoreExecutions || executions.length === 0) return;+ const oldestId = executions[executions.length - 1]?.id;+ if (oldestId == null) return;++ setLoadingMoreExecutions(true);+ try {+ const res = await fetch(pipeExecutionsUrl(apiBase, name, oldestId));+ const data = await res.json();+ const olderExecutions: PipeExecution[] = data.data || [];+ setExecutions((prev) => {+ const seen = new Set(prev.map((exec) => exec.id));+ return [+ ...prev,+ ...olderExecutions.filter((exec) => !seen.has(exec.id)),+ ];+ });+ setHasMoreExecutions(olderExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);+ } catch (e) {+ console.error("failed to fetch older executions:", e);+ } finally {+ setLoadingMoreExecutions(false);+ }+ };+const togglePipe = async (name: string, enabled: boolean) => {posthog.capture("pipe_toggled", { pipe: name, enabled });// Optimistic update — flip the switch immediately@@ -2797,7 +2842,8 @@ export function PipesSection() {no runs yet — click ▶ to run manually</p>) : executions.length > 0 ? (- executions.map((exec) => (+ <>+ {executions.map((exec) => (// contain: layout paint isolates the markdown// subtree's reflow cost from page-wide layout// passes. Without it, opening the device /@@ -2866,8 +2912,25 @@ export function PipesSection() {</pre>);})()}- </div>- ))+ </div>+ ))}+ {hasMoreExecutions && (+ <Button+ variant="ghost"+ size="sm"+ className="w-full h-8 text-xs"+ disabled={loadingMoreExecutions}+ onClick={() => loadMoreExecutions(pipe.config.name)}+ >+ {loadingMoreExecutions ? (+ <Loader2 className="h-3.5 w-3.5 mr-2 animate-spin" />+ ) : (+ <ChevronDown className="h-3.5 w-3.5 mr-2" />+ )}+ load older runs+ </Button>+ )}+ </>) : (logs.slice().reverse().map((log, i) => (// see contain: layout paint comment abovediff --git a/crates/screenpipe-core/src/pipes/mod.rs b/crates/screenpipe-core/src/pipes/mod.rsindex cf386f7d2..d0dc19989 100644--- a/crates/screenpipe-core/src/pipes/mod.rs+++ b/crates/screenpipe-core/src/pipes/mod.rs@@ -43,6 +43,7 @@ const PIPE_COMPLETED_EVENT_PREFIX: &str = "pipe_completed:";const PIPE_LOG_ACTIVE_KEEP_PER_PIPE: usize = 200;const PIPE_LOG_ARCHIVE_AFTER_DAYS: i64 = 14;const PIPE_LOG_ARCHIVE_DIR: &str = "archive";+const PIPE_EXECUTION_KEEP_PER_PIPE: i32 = 500;// ---------------------------------------------------------------------------// Config & log types@@ -1103,7 +1104,12 @@ pub trait PipeStore: Send + Sync {) -> Result<()>;/// Get recent executions for a pipe (newest first).- async fn get_executions(&self, pipe_name: &str, limit: i32) -> Result<Vec<PipeExecution>>;+ async fn get_executions(+ &self,+ pipe_name: &str,+ limit: i32,+ before_id: Option<i64>,+ ) -> Result<Vec<PipeExecution>>;/// Mark any 'running' executions as failed (orphan recovery on startup)./// Returns the number of rows updated.@@ -1978,7 +1984,7 @@ pub struct PipeManager {/// Loaded pipe configs keyed by pipe name: (config, prompt_body, raw_content).#[allow(clippy::type_complexity)]pipes: Arc<Mutex<HashMap<String, (PipeConfig, String, String)>>>,- /// Recent run logs per pipe (last 50).+ /// Recent run logs per pipe.logs: Arc<Mutex<HashMap<String, VecDeque<PipeRunLog>>>>,/// Currently running pipe PIDs.running: Arc<Mutex<HashMap<String, ExecutionHandle>>>,@@ -2175,10 +2181,13 @@ impl PipeManager {);}- /// Delete old pipe executions, keeping only the newest 50 per pipe.+ /// Delete old pipe executions, keeping enough recent history for the UI.pub async fn cleanup_executions(&self) {if let Some(ref store) = self.store {- match store.cleanup_old_executions(50).await {+ match store+ .cleanup_old_executions(PIPE_EXECUTION_KEEP_PER_PIPE)+ .await+ {Ok(count) => {if count > 0 {info!("pipe cleanup: deleted {} old executions", count);@@ -2390,6 +2399,15 @@ impl PipeManager {for (name, mut status) in partial {if let Some(state) = states.get(&name) {status.consecutive_failures = state.consecutive_failures;+ if state+ .last_run_at+ .zip(status.last_run)+ .map(|(persisted, in_memory)| persisted > in_memory)+ .unwrap_or(status.last_run.is_none() && state.last_run_at.is_some())+ {+ status.last_run = state.last_run_at;+ status.last_success = Some(state.consecutive_failures == 0);+ }}result.push(status);}@@ -2546,6 +2564,15 @@ impl PipeManager {if let Some(ref store) = self.store {if let Ok(Some(state)) = store.get_scheduler_state(name).await {status.consecutive_failures = state.consecutive_failures;+ if state+ .last_run_at+ .zip(status.last_run)+ .map(|(persisted, in_memory)| persisted > in_memory)+ .unwrap_or(status.last_run.is_none() && state.last_run_at.is_some())+ {+ status.last_run = state.last_run_at;+ status.last_success = Some(state.consecutive_failures == 0);+ }}}Some(status)@@ -2560,7 +2587,10 @@ impl PipeManager {// Prefer database — it is the persistent source of truth and// survives process restarts (the in-memory HashMap does not).if let Some(ref store) = self.store {- if let Ok(executions) = store.get_executions(name, 50).await {+ if let Ok(executions) = store+ .get_executions(name, PIPE_LOG_ACTIVE_KEEP_PER_PIPE as i32, None)+ .await+ {let db_logs: Vec<PipeRunLog> = executions.into_iter().filter(|e| e.status != "queued" && e.status != "running")@@ -2600,9 +2630,14 @@ impl PipeManager {}/// Get execution history from the DB store.- pub async fn get_executions(&self, name: &str, limit: i32) -> Result<Vec<PipeExecution>> {+ pub async fn get_executions(+ &self,+ name: &str,+ limit: i32,+ before_id: Option<i64>,+ ) -> Result<Vec<PipeExecution>> {if let Some(ref store) = self.store {- store.get_executions(name, limit).await+ store.get_executions(name, limit, before_id).await} else {Ok(vec![])}@@ -3142,7 +3177,7 @@ impl PipeManager {let mut l = logs_ref.lock().await;let entry = l.entry(log.pipe_name.clone()).or_insert_with(VecDeque::new);entry.push_back(log);- if entry.len() > 50 {+ if entry.len() > PIPE_LOG_ACTIVE_KEEP_PER_PIPE {entry.pop_front();}drop(l);@@ -4508,7 +4543,7 @@ impl PipeManager {stdout: String::new(),stderr: message.clone(),});- if entry.len() > 50 {+ if entry.len() > PIPE_LOG_ACTIVE_KEEP_PER_PIPE {entry.pop_back();}}@@ -5048,7 +5083,7 @@ impl PipeManager {let mut l = logs_ref.lock().await;let entry = l.entry(log.pipe_name.clone()).or_insert_with(VecDeque::new);entry.push_back(log);- if entry.len() > 50 {+ if entry.len() > PIPE_LOG_ACTIVE_KEEP_PER_PIPE {entry.pop_front();}drop(l);@@ -5106,7 +5141,10 @@ impl PipeManager {// Daily cleanup: prune old executions and archive old disk logs every 24hif last_cleanup.elapsed() >= std::time::Duration::from_secs(86400) {if let Some(ref store) = store {- match store.cleanup_old_executions(50).await {+ match store+ .cleanup_old_executions(PIPE_EXECUTION_KEEP_PER_PIPE)+ .await+ {Ok(count) if count > 0 => {info!("scheduler cleanup: deleted {} old executions", count);}@@ -5378,7 +5416,7 @@ impl PipeManager {let mut logs = self.logs.lock().await;let entry = logs.entry(name.to_string()).or_insert_with(VecDeque::new);entry.push_back(log.clone());- if entry.len() > 50 {+ if entry.len() > PIPE_LOG_ACTIVE_KEEP_PER_PIPE {entry.pop_front();}}diff --git a/crates/screenpipe-engine/src/pipe_store.rs b/crates/screenpipe-engine/src/pipe_store.rsindex 92b4dc2e6..af8370032 100644--- a/crates/screenpipe-engine/src/pipe_store.rs+++ b/crates/screenpipe-engine/src/pipe_store.rs@@ -106,20 +106,42 @@ impl PipeStore for SqlitePipeStore {Ok(())}- async fn get_executions(&self, pipe_name: &str, limit: i32) -> Result<Vec<PipeExecution>> {- let rows = sqlx::query_as::<_, PipeExecutionRow>(- r#"SELECT id, pipe_name, status, trigger_type, pid, model, provider,- started_at, finished_at, stdout, stderr, exit_code,- error_type, error_message, duration_ms, session_path- FROM pipe_executions- WHERE pipe_name = ?- ORDER BY id DESC- LIMIT ?"#,- )- .bind(pipe_name)- .bind(limit)- .fetch_all(&self.db.pool)- .await?;+ async fn get_executions(+ &self,+ pipe_name: &str,+ limit: i32,+ before_id: Option<i64>,+ ) -> Result<Vec<PipeExecution>> {+ let rows = if let Some(before_id) = before_id {+ sqlx::query_as::<_, PipeExecutionRow>(+ r#"SELECT id, pipe_name, status, trigger_type, pid, model, provider,+ started_at, finished_at, stdout, stderr, exit_code,+ error_type, error_message, duration_ms, session_path+ FROM pipe_executions+ WHERE pipe_name = ? AND id < ?+ ORDER BY id DESC+ LIMIT ?"#,+ )+ .bind(pipe_name)+ .bind(before_id)+ .bind(limit)+ .fetch_all(&self.db.pool)+ .await?+ } else {+ sqlx::query_as::<_, PipeExecutionRow>(+ r#"SELECT id, pipe_name, status, trigger_type, pid, model, provider,+ started_at, finished_at, stdout, stderr, exit_code,+ error_type, error_message, duration_ms, session_path+ FROM pipe_executions+ WHERE pipe_name = ?+ ORDER BY id DESC+ LIMIT ?"#,+ )+ .bind(pipe_name)+ .bind(limit)+ .fetch_all(&self.db.pool)+ .await?+ };Ok(rows.into_iter().map(|r| r.into()).collect())}@@ -468,7 +490,7 @@ mod tests {.await.unwrap();- let execs = store.get_executions("test-pipe", 10).await.unwrap();+ let execs = store.get_executions("test-pipe", 10, None).await.unwrap();assert_eq!(execs.len(), 1);assert_eq!(execs[0].id, id);assert_eq!(execs[0].pipe_name, "test-pipe");@@ -488,7 +510,7 @@ mod tests {store.set_execution_running(id, Some(12345)).await.unwrap();- let execs = store.get_executions("p", 10).await.unwrap();+ let execs = store.get_executions("p", 10, None).await.unwrap();assert_eq!(execs[0].status, "running");assert_eq!(execs[0].pid, Some(12345));}@@ -515,7 +537,7 @@ mod tests {.await.unwrap();- let execs = store.get_executions("p", 10).await.unwrap();+ let execs = store.get_executions("p", 10, None).await.unwrap();assert_eq!(execs[0].status, "completed");assert_eq!(execs[0].stdout, "hello stdout");assert_eq!(execs[0].exit_code, Some(0));@@ -545,7 +567,7 @@ mod tests {.await.unwrap();- let execs = store.get_executions("p", 10).await.unwrap();+ let execs = store.get_executions("p", 10, None).await.unwrap();assert_eq!(execs[0].status, "failed");assert_eq!(execs[0].error_type.as_deref(), Some("rate_limited"));assert_eq!(@@ -576,7 +598,7 @@ mod tests {.await.unwrap();- let execs = store.get_executions("p", 10).await.unwrap();+ let execs = store.get_executions("p", 10, None).await.unwrap();assert_eq!(execs[0].status, "timed_out");assert_eq!(execs[0].error_type.as_deref(), Some("timeout"));}@@ -591,12 +613,34 @@ mod tests {.unwrap();}- let execs = store.get_executions("p", 3).await.unwrap();+ let execs = store.get_executions("p", 3, None).await.unwrap();assert_eq!(execs.len(), 3);// Should be newest first (descending by id)assert!(execs[0].id > execs[1].id);}+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn test_get_executions_before_id() {+ let (store, _tmp) = setup_test_store().await;+ for _ in 0..5 {+ store+ .create_execution("p", "manual", "m", None)+ .await+ .unwrap();+ }++ let first_page = store.get_executions("p", 2, None).await.unwrap();+ assert_eq!(first_page.len(), 2);++ let second_page = store+ .get_executions("p", 2, Some(first_page[1].id))+ .await+ .unwrap();+ assert_eq!(second_page.len(), 2);+ assert!(second_page.iter().all(|exec| exec.id < first_page[1].id));+ assert!(second_page[0].id > second_page[1].id);+ }+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn test_get_executions_filters_by_pipe_name() {let (store, _tmp) = setup_test_store().await;@@ -613,8 +657,8 @@ mod tests {.await.unwrap();- let execs_a = store.get_executions("pipe-a", 10).await.unwrap();- let execs_b = store.get_executions("pipe-b", 10).await.unwrap();+ let execs_a = store.get_executions("pipe-a", 10, None).await.unwrap();+ let execs_b = store.get_executions("pipe-b", 10, None).await.unwrap();assert_eq!(execs_a.len(), 2);assert_eq!(execs_b.len(), 1);}@@ -653,15 +697,15 @@ mod tests {let _count = store.mark_orphaned_running().await.unwrap();// Verify states- let execs = store.get_executions("p1", 10).await.unwrap();+ let execs = store.get_executions("p1", 10, None).await.unwrap();assert_eq!(execs[0].status, "failed");assert_eq!(execs[0].error_type.as_deref(), Some("interrupted"));- let execs = store.get_executions("p2", 10).await.unwrap();+ let execs = store.get_executions("p2", 10, None).await.unwrap();assert_eq!(execs[0].status, "failed");// p3 should be unaffected- let execs = store.get_executions("p3", 10).await.unwrap();+ let execs = store.get_executions("p3", 10, None).await.unwrap();assert_eq!(execs[0].status, "completed");}@@ -755,10 +799,10 @@ mod tests {// by checking the remaining execution counts below.let _deleted = store.cleanup_old_executions(2).await.unwrap();- let execs_a = store.get_executions("pipe-a", 10).await.unwrap();+ let execs_a = store.get_executions("pipe-a", 10, None).await.unwrap();assert_eq!(execs_a.len(), 2);- let execs_b = store.get_executions("pipe-b", 10).await.unwrap();+ let execs_b = store.get_executions("pipe-b", 10, None).await.unwrap();assert_eq!(execs_b.len(), 2);}@@ -773,7 +817,7 @@ mod tests {let deleted = store.cleanup_old_executions(50).await.unwrap();assert_eq!(deleted, 0);- let execs = store.get_executions("pipe-a", 10).await.unwrap();+ let execs = store.get_executions("pipe-a", 10, None).await.unwrap();assert_eq!(execs.len(), 1);}@@ -792,13 +836,19 @@ mod tests {.await.unwrap();- let execs = store.get_executions("lifecycle-pipe", 10).await.unwrap();+ let execs = store+ .get_executions("lifecycle-pipe", 10, None)+ .await+ .unwrap();assert_eq!(execs[0].status, "queued");// 2. Running with PIDstore.set_execution_running(id, Some(42)).await.unwrap();- let execs = store.get_executions("lifecycle-pipe", 10).await.unwrap();+ let execs = store+ .get_executions("lifecycle-pipe", 10, None)+ .await+ .unwrap();assert_eq!(execs[0].status, "running");assert_eq!(execs[0].pid, Some(42));@@ -817,7 +867,10 @@ mod tests {.await.unwrap();- let execs = store.get_executions("lifecycle-pipe", 10).await.unwrap();+ let execs = store+ .get_executions("lifecycle-pipe", 10, None)+ .await+ .unwrap();assert_eq!(execs[0].status, "completed");assert_eq!(execs[0].stdout, "result output");assert!(execs[0].duration_ms.unwrap() >= 0);diff --git a/crates/screenpipe-engine/src/pipes_api.rs b/crates/screenpipe-engine/src/pipes_api.rsindex 09db4fe10..eb6f26ba2 100644--- a/crates/screenpipe-engine/src/pipes_api.rs+++ b/crates/screenpipe-engine/src/pipes_api.rs@@ -51,11 +51,13 @@ pub struct ConfigUpdateRequest {#[derive(Deserialize)]pub struct ExecutionsQuery {pub limit: Option<i32>,+ pub before_id: Option<i64>,}#[derive(Deserialize)]pub struct ListPipesQuery {pub include_executions: Option<bool>,+ pub execution_limit: Option<i32>,}// ---------------------------------------------------------------------------@@ -74,7 +76,8 @@ pub async fn list_pipes(tracing::warn!("failed to reload pipes from disk: {}", e);}if query.include_executions.unwrap_or(false) {- let pipes_with_execs = mgr.list_pipes_with_executions(5).await;+ let execution_limit = query.execution_limit.unwrap_or(5).clamp(1, 100);+ let pipes_with_execs = mgr.list_pipes_with_executions(execution_limit).await;let total = pipes_with_execs.len();let data: Vec<Value> = pipes_with_execs.into_iter()@@ -267,8 +270,8 @@ pub async fn get_pipe_executions(Query(query): Query<ExecutionsQuery>,) -> Json<Value> {let mgr = pm.lock().await;- let limit = query.limit.unwrap_or(20).min(100);- match mgr.get_executions(&id, limit).await {+ let limit = query.limit.unwrap_or(100).clamp(1, 500);+ match mgr.get_executions(&id, limit, query.before_id).await {Ok(executions) => Json(json!({ "data": executions })),Err(e) => Json(json!({ "error": e.to_string() })),}@@ -280,7 +283,7 @@ pub async fn get_pipe_session(Path((id, exec_id)): Path<(String, i64)>,) -> (StatusCode, Json<Value>) {let mgr = pm.lock().await;- let execs = match mgr.get_executions(&id, 100).await {+ let execs = match mgr.get_executions(&id, 1, exec_id.checked_add(1)).await {Ok(e) => e,Err(e) => {return (