Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Fix pipe run history pagination

Louis Beaumont152be2ad0a43
diff --git a/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx b/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx
index 9938c49b1..9b86961e6 100644
--- a/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx
@@ -129,6 +129,18 @@ import { useDeviceMonitor } from "@/lib/hooks/use-device-monitor";
import { Monitor, Wifi, WifiOff, ScanSearch } from "lucide-react";
import { requestPipeStop } from "@/lib/pipe-stop";
+const PIPE_EXECUTIONS_PAGE_LIMIT = 100;
+
+function pipeExecutionsUrl(apiBase: string, pipeName: string, beforeId?: number) {
+ const params = new URLSearchParams({
+ limit: String(PIPE_EXECUTIONS_PAGE_LIMIT),
+ });
+ if (beforeId != null) {
+ params.set("before_id", String(beforeId));
+ }
+ return `${apiBase}/pipes/${encodeURIComponent(pipeName)}/executions?${params.toString()}`;
+}
+
const PIPE_CREATION_PROMPT = `create a screenpipe pipe that does the following.
## what is screenpipe?
@@ -1018,6 +1030,8 @@ export function PipesSection() {
const [logs, setLogs] = useState<PipeRunLog[]>([]);
const [executions, setExecutions] = useState<PipeExecution[]>([]);
const [executionsLoading, setExecutionsLoading] = useState(false);
+ const [hasMoreExecutions, setHasMoreExecutions] = useState(false);
+ const [loadingMoreExecutions, setLoadingMoreExecutions] = useState(false);
// Per-pipe recent executions (always fetched for all pipes)
const [pipeExecutions, setPipeExecutions] = useState<Record<string, PipeExecution[]>>({});
const [loading, setLoading] = useState(true);
@@ -1640,9 +1654,11 @@ export function PipesSection() {
const exp = expandedRef.current;
if (exp) {
try {
- const execRes = await fetch(`${apiBase}/pipes/${exp}/executions?limit=20`);
+ const execRes = await fetch(pipeExecutionsUrl(apiBase, exp));
const execData = await execRes.json();
- setExecutions(execData.data || []);
+ const nextExecutions = execData.data || [];
+ setExecutions(nextExecutions);
+ setHasMoreExecutions(nextExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);
const finishedKeys = (execData.data || [])
.filter((e: PipeExecution) => e.status !== "running")
.map((e: PipeExecution) => `${e.pipe_name}:${e.id}`);
@@ -1688,18 +1704,47 @@ export function PipesSection() {
const fetchExecutions = async (name: string) => {
setExecutionsLoading(true);
+ setHasMoreExecutions(false);
try {
- const res = await fetch(`${apiBase}/pipes/${name}/executions?limit=20`);
+ const res = await fetch(pipeExecutionsUrl(apiBase, name));
const data = await res.json();
- setExecutions(data.data || []);
+ const nextExecutions = data.data || [];
+ setExecutions(nextExecutions);
+ setHasMoreExecutions(nextExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);
} catch (e) {
// Executions endpoint may not exist on older servers — fall back silently
setExecutions([]);
+ setHasMoreExecutions(false);
} finally {
setExecutionsLoading(false);
}
};
+ const loadMoreExecutions = async (name: string) => {
+ if (loadingMoreExecutions || executions.length === 0) return;
+ const oldestId = executions[executions.length - 1]?.id;
+ if (oldestId == null) return;
+
+ setLoadingMoreExecutions(true);
+ try {
+ const res = await fetch(pipeExecutionsUrl(apiBase, name, oldestId));
+ const data = await res.json();
+ const olderExecutions: PipeExecution[] = data.data || [];
+ setExecutions((prev) => {
+ const seen = new Set(prev.map((exec) => exec.id));
+ return [
+ ...prev,
+ ...olderExecutions.filter((exec) => !seen.has(exec.id)),
+ ];
+ });
+ setHasMoreExecutions(olderExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);
+ } catch (e) {
+ console.error("failed to fetch older executions:", e);
+ } finally {
+ setLoadingMoreExecutions(false);
+ }
+ };
+
const togglePipe = async (name: string, enabled: boolean) => {
posthog.capture("pipe_toggled", { pipe: name, enabled });
// Optimistic update — flip the switch immediately
@@ -2797,7 +2842,8 @@ export function PipesSection() {
no runs yet — click ▶ to run manually
</p>
) : executions.length > 0 ? (
- executions.map((exec) => (
+ <>
+ {executions.map((exec) => (
// contain: layout paint isolates the markdown
// subtree's reflow cost from page-wide layout
// passes. Without it, opening the device /
@@ -2866,8 +2912,25 @@ export function PipesSection() {
</pre>
);
})()}
- </div>
- ))
+ </div>
+ ))}
+ {hasMoreExecutions && (
+ <Button
+ variant="ghost"
+ size="sm"
+ className="w-full h-8 text-xs"
+ disabled={loadingMoreExecutions}
+ onClick={() => loadMoreExecutions(pipe.config.name)}
+ >
+ {loadingMoreExecutions ? (
+ <Loader2 className="h-3.5 w-3.5 mr-2 animate-spin" />
+ ) : (
+ <ChevronDown className="h-3.5 w-3.5 mr-2" />
+ )}
+ load older runs
+ </Button>
+ )}
+ </>
) : (
logs.slice().reverse().map((log, i) => (
// see contain: layout paint comment above
diff --git a/crates/screenpipe-core/src/pipes/mod.rs b/crates/screenpipe-core/src/pipes/mod.rs
index cf386f7d2..d0dc19989 100644
--- a/crates/screenpipe-core/src/pipes/mod.rs
+++ b/crates/screenpipe-core/src/pipes/mod.rs
@@ -43,6 +43,7 @@ const PIPE_COMPLETED_EVENT_PREFIX: &str = "pipe_completed:";
const PIPE_LOG_ACTIVE_KEEP_PER_PIPE: usize = 200;
const PIPE_LOG_ARCHIVE_AFTER_DAYS: i64 = 14;
const PIPE_LOG_ARCHIVE_DIR: &str = "archive";
+const PIPE_EXECUTION_KEEP_PER_PIPE: i32 = 500;
// ---------------------------------------------------------------------------
// Config & log types
@@ -1103,7 +1104,12 @@ pub trait PipeStore: Send + Sync {
) -> Result<()>;
/// Get recent executions for a pipe (newest first).
- async fn get_executions(&self, pipe_name: &str, limit: i32) -> Result<Vec<PipeExecution>>;
+ async fn get_executions(
+ &self,
+ pipe_name: &str,
+ limit: i32,
+ before_id: Option<i64>,
+ ) -> Result<Vec<PipeExecution>>;
/// Mark any 'running' executions as failed (orphan recovery on startup).
/// Returns the number of rows updated.
@@ -1978,7 +1984,7 @@ pub struct PipeManager {
/// Loaded pipe configs keyed by pipe name: (config, prompt_body, raw_content).
#[allow(clippy::type_complexity)]
pipes: Arc<Mutex<HashMap<String, (PipeConfig, String, String)>>>,
- /// Recent run logs per pipe (last 50).
+ /// Recent run logs per pipe.
logs: Arc<Mutex<HashMap<String, VecDeque<PipeRunLog>>>>,
/// Currently running pipe PIDs.
running: Arc<Mutex<HashMap<String, ExecutionHandle>>>,
@@ -2175,10 +2181,13 @@ impl PipeManager {
);
}
- /// Delete old pipe executions, keeping only the newest 50 per pipe.
+ /// Delete old pipe executions, keeping enough recent history for the UI.
pub async fn cleanup_executions(&self) {
if let Some(ref store) = self.store {
- match store.cleanup_old_executions(50).await {
+ match store
+ .cleanup_old_executions(PIPE_EXECUTION_KEEP_PER_PIPE)
+ .await
+ {
Ok(count) => {
if count > 0 {
info!("pipe cleanup: deleted {} old executions", count);
@@ -2390,6 +2399,15 @@ impl PipeManager {
for (name, mut status) in partial {
if let Some(state) = states.get(&name) {
status.consecutive_failures = state.consecutive_failures;
+ if state
+ .last_run_at
+ .zip(status.last_run)
+ .map(|(persisted, in_memory)| persisted > in_memory)
+ .unwrap_or(status.last_run.is_none() && state.last_run_at.is_some())
+ {
+ status.last_run = state.last_run_at;
+ status.last_success = Some(state.consecutive_failures == 0);
+ }
}
result.push(status);
}
@@ -2546,6 +2564,15 @@ impl PipeManager {
if let Some(ref store) = self.store {
if let Ok(Some(state)) = store.get_scheduler_state(name).await {
status.consecutive_failures = state.consecutive_failures;
+ if state
+ .last_run_at
+ .zip(status.last_run)
+ .map(|(persisted, in_memory)| persisted > in_memory)
+ .unwrap_or(status.last_run.is_none() && state.last_run_at.is_some())
+ {
+ status.last_run = state.last_run_at;
+ status.last_success = Some(state.consecutive_failures == 0);
+ }
}
}
Some(status)
@@ -2560,7 +2587,10 @@ impl PipeManager {
// Prefer database — it is the persistent source of truth and
// survives process restarts (the in-memory HashMap does not).
if let Some(ref store) = self.store {
- if let Ok(executions) = store.get_executions(name, 50).await {
+ if let Ok(executions) = store
+ .get_executions(name, PIPE_LOG_ACTIVE_KEEP_PER_PIPE as i32, None)
+ .await
+ {
let db_logs: Vec<PipeRunLog> = executions
.into_iter()
.filter(|e| e.status != "queued" && e.status != "running")
@@ -2600,9 +2630,14 @@ impl PipeManager {
}
/// Get execution history from the DB store.
- pub async fn get_executions(&self, name: &str, limit: i32) -> Result<Vec<PipeExecution>> {
+ pub async fn get_executions(
+ &self,
+ name: &str,
+ limit: i32,
+ before_id: Option<i64>,
+ ) -> Result<Vec<PipeExecution>> {
if let Some(ref store) = self.store {
- store.get_executions(name, limit).await
+ store.get_executions(name, limit, before_id).await
} else {
Ok(vec![])
}
@@ -3142,7 +3177,7 @@ impl PipeManager {
let mut l = logs_ref.lock().await;
let entry = l.entry(log.pipe_name.clone()).or_insert_with(VecDeque::new);
entry.push_back(log);
- if entry.len() > 50 {
+ if entry.len() > PIPE_LOG_ACTIVE_KEEP_PER_PIPE {
entry.pop_front();
}
drop(l);
@@ -4508,7 +4543,7 @@ impl PipeManager {
stdout: String::new(),
stderr: message.clone(),
});
- if entry.len() > 50 {
+ if entry.len() > PIPE_LOG_ACTIVE_KEEP_PER_PIPE {
entry.pop_back();
}
}
@@ -5048,7 +5083,7 @@ impl PipeManager {
let mut l = logs_ref.lock().await;
let entry = l.entry(log.pipe_name.clone()).or_insert_with(VecDeque::new);
entry.push_back(log);
- if entry.len() > 50 {
+ if entry.len() > PIPE_LOG_ACTIVE_KEEP_PER_PIPE {
entry.pop_front();
}
drop(l);
@@ -5106,7 +5141,10 @@ impl PipeManager {
// Daily cleanup: prune old executions and archive old disk logs every 24h
if last_cleanup.elapsed() >= std::time::Duration::from_secs(86400) {
if let Some(ref store) = store {
- match store.cleanup_old_executions(50).await {
+ match store
+ .cleanup_old_executions(PIPE_EXECUTION_KEEP_PER_PIPE)
+ .await
+ {
Ok(count) if count > 0 => {
info!("scheduler cleanup: deleted {} old executions", count);
}
@@ -5378,7 +5416,7 @@ impl PipeManager {
let mut logs = self.logs.lock().await;
let entry = logs.entry(name.to_string()).or_insert_with(VecDeque::new);
entry.push_back(log.clone());
- if entry.len() > 50 {
+ if entry.len() > PIPE_LOG_ACTIVE_KEEP_PER_PIPE {
entry.pop_front();
}
}
diff --git a/crates/screenpipe-engine/src/pipe_store.rs b/crates/screenpipe-engine/src/pipe_store.rs
index 92b4dc2e6..af8370032 100644
--- a/crates/screenpipe-engine/src/pipe_store.rs
+++ b/crates/screenpipe-engine/src/pipe_store.rs
@@ -106,20 +106,42 @@ impl PipeStore for SqlitePipeStore {
Ok(())
}
- async fn get_executions(&self, pipe_name: &str, limit: i32) -> Result<Vec<PipeExecution>> {
- let rows = sqlx::query_as::<_, PipeExecutionRow>(
- r#"SELECT id, pipe_name, status, trigger_type, pid, model, provider,
- started_at, finished_at, stdout, stderr, exit_code,
- error_type, error_message, duration_ms, session_path
- FROM pipe_executions
- WHERE pipe_name = ?
- ORDER BY id DESC
- LIMIT ?"#,
- )
- .bind(pipe_name)
- .bind(limit)
- .fetch_all(&self.db.pool)
- .await?;
+ async fn get_executions(
+ &self,
+ pipe_name: &str,
+ limit: i32,
+ before_id: Option<i64>,
+ ) -> Result<Vec<PipeExecution>> {
+ let rows = if let Some(before_id) = before_id {
+ sqlx::query_as::<_, PipeExecutionRow>(
+ r#"SELECT id, pipe_name, status, trigger_type, pid, model, provider,
+ started_at, finished_at, stdout, stderr, exit_code,
+ error_type, error_message, duration_ms, session_path
+ FROM pipe_executions
+ WHERE pipe_name = ? AND id < ?
+ ORDER BY id DESC
+ LIMIT ?"#,
+ )
+ .bind(pipe_name)
+ .bind(before_id)
+ .bind(limit)
+ .fetch_all(&self.db.pool)
+ .await?
+ } else {
+ sqlx::query_as::<_, PipeExecutionRow>(
+ r#"SELECT id, pipe_name, status, trigger_type, pid, model, provider,
+ started_at, finished_at, stdout, stderr, exit_code,
+ error_type, error_message, duration_ms, session_path
+ FROM pipe_executions
+ WHERE pipe_name = ?
+ ORDER BY id DESC
+ LIMIT ?"#,
+ )
+ .bind(pipe_name)
+ .bind(limit)
+ .fetch_all(&self.db.pool)
+ .await?
+ };
Ok(rows.into_iter().map(|r| r.into()).collect())
}
@@ -468,7 +490,7 @@ mod tests {
.await
.unwrap();
- let execs = store.get_executions("test-pipe", 10).await.unwrap();
+ let execs = store.get_executions("test-pipe", 10, None).await.unwrap();
assert_eq!(execs.len(), 1);
assert_eq!(execs[0].id, id);
assert_eq!(execs[0].pipe_name, "test-pipe");
@@ -488,7 +510,7 @@ mod tests {
store.set_execution_running(id, Some(12345)).await.unwrap();
- let execs = store.get_executions("p", 10).await.unwrap();
+ let execs = store.get_executions("p", 10, None).await.unwrap();
assert_eq!(execs[0].status, "running");
assert_eq!(execs[0].pid, Some(12345));
}
@@ -515,7 +537,7 @@ mod tests {
.await
.unwrap();
- let execs = store.get_executions("p", 10).await.unwrap();
+ let execs = store.get_executions("p", 10, None).await.unwrap();
assert_eq!(execs[0].status, "completed");
assert_eq!(execs[0].stdout, "hello stdout");
assert_eq!(execs[0].exit_code, Some(0));
@@ -545,7 +567,7 @@ mod tests {
.await
.unwrap();
- let execs = store.get_executions("p", 10).await.unwrap();
+ let execs = store.get_executions("p", 10, None).await.unwrap();
assert_eq!(execs[0].status, "failed");
assert_eq!(execs[0].error_type.as_deref(), Some("rate_limited"));
assert_eq!(
@@ -576,7 +598,7 @@ mod tests {
.await
.unwrap();
- let execs = store.get_executions("p", 10).await.unwrap();
+ let execs = store.get_executions("p", 10, None).await.unwrap();
assert_eq!(execs[0].status, "timed_out");
assert_eq!(execs[0].error_type.as_deref(), Some("timeout"));
}
@@ -591,12 +613,34 @@ mod tests {
.unwrap();
}
- let execs = store.get_executions("p", 3).await.unwrap();
+ let execs = store.get_executions("p", 3, None).await.unwrap();
assert_eq!(execs.len(), 3);
// Should be newest first (descending by id)
assert!(execs[0].id > execs[1].id);
}
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn test_get_executions_before_id() {
+ let (store, _tmp) = setup_test_store().await;
+ for _ in 0..5 {
+ store
+ .create_execution("p", "manual", "m", None)
+ .await
+ .unwrap();
+ }
+
+ let first_page = store.get_executions("p", 2, None).await.unwrap();
+ assert_eq!(first_page.len(), 2);
+
+ let second_page = store
+ .get_executions("p", 2, Some(first_page[1].id))
+ .await
+ .unwrap();
+ assert_eq!(second_page.len(), 2);
+ assert!(second_page.iter().all(|exec| exec.id < first_page[1].id));
+ assert!(second_page[0].id > second_page[1].id);
+ }
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_get_executions_filters_by_pipe_name() {
let (store, _tmp) = setup_test_store().await;
@@ -613,8 +657,8 @@ mod tests {
.await
.unwrap();
- let execs_a = store.get_executions("pipe-a", 10).await.unwrap();
- let execs_b = store.get_executions("pipe-b", 10).await.unwrap();
+ let execs_a = store.get_executions("pipe-a", 10, None).await.unwrap();
+ let execs_b = store.get_executions("pipe-b", 10, None).await.unwrap();
assert_eq!(execs_a.len(), 2);
assert_eq!(execs_b.len(), 1);
}
@@ -653,15 +697,15 @@ mod tests {
let _count = store.mark_orphaned_running().await.unwrap();
// Verify states
- let execs = store.get_executions("p1", 10).await.unwrap();
+ let execs = store.get_executions("p1", 10, None).await.unwrap();
assert_eq!(execs[0].status, "failed");
assert_eq!(execs[0].error_type.as_deref(), Some("interrupted"));
- let execs = store.get_executions("p2", 10).await.unwrap();
+ let execs = store.get_executions("p2", 10, None).await.unwrap();
assert_eq!(execs[0].status, "failed");
// p3 should be unaffected
- let execs = store.get_executions("p3", 10).await.unwrap();
+ let execs = store.get_executions("p3", 10, None).await.unwrap();
assert_eq!(execs[0].status, "completed");
}
@@ -755,10 +799,10 @@ mod tests {
// by checking the remaining execution counts below.
let _deleted = store.cleanup_old_executions(2).await.unwrap();
- let execs_a = store.get_executions("pipe-a", 10).await.unwrap();
+ let execs_a = store.get_executions("pipe-a", 10, None).await.unwrap();
assert_eq!(execs_a.len(), 2);
- let execs_b = store.get_executions("pipe-b", 10).await.unwrap();
+ let execs_b = store.get_executions("pipe-b", 10, None).await.unwrap();
assert_eq!(execs_b.len(), 2);
}
@@ -773,7 +817,7 @@ mod tests {
let deleted = store.cleanup_old_executions(50).await.unwrap();
assert_eq!(deleted, 0);
- let execs = store.get_executions("pipe-a", 10).await.unwrap();
+ let execs = store.get_executions("pipe-a", 10, None).await.unwrap();
assert_eq!(execs.len(), 1);
}
@@ -792,13 +836,19 @@ mod tests {
.await
.unwrap();
- let execs = store.get_executions("lifecycle-pipe", 10).await.unwrap();
+ let execs = store
+ .get_executions("lifecycle-pipe", 10, None)
+ .await
+ .unwrap();
assert_eq!(execs[0].status, "queued");
// 2. Running with PID
store.set_execution_running(id, Some(42)).await.unwrap();
- let execs = store.get_executions("lifecycle-pipe", 10).await.unwrap();
+ let execs = store
+ .get_executions("lifecycle-pipe", 10, None)
+ .await
+ .unwrap();
assert_eq!(execs[0].status, "running");
assert_eq!(execs[0].pid, Some(42));
@@ -817,7 +867,10 @@ mod tests {
.await
.unwrap();
- let execs = store.get_executions("lifecycle-pipe", 10).await.unwrap();
+ let execs = store
+ .get_executions("lifecycle-pipe", 10, None)
+ .await
+ .unwrap();
assert_eq!(execs[0].status, "completed");
assert_eq!(execs[0].stdout, "result output");
assert!(execs[0].duration_ms.unwrap() >= 0);
diff --git a/crates/screenpipe-engine/src/pipes_api.rs b/crates/screenpipe-engine/src/pipes_api.rs
index 09db4fe10..eb6f26ba2 100644
--- a/crates/screenpipe-engine/src/pipes_api.rs
+++ b/crates/screenpipe-engine/src/pipes_api.rs
@@ -51,11 +51,13 @@ pub struct ConfigUpdateRequest {
#[derive(Deserialize)]
pub struct ExecutionsQuery {
pub limit: Option<i32>,
+ pub before_id: Option<i64>,
}
#[derive(Deserialize)]
pub struct ListPipesQuery {
pub include_executions: Option<bool>,
+ pub execution_limit: Option<i32>,
}
// ---------------------------------------------------------------------------
@@ -74,7 +76,8 @@ pub async fn list_pipes(
tracing::warn!("failed to reload pipes from disk: {}", e);
}
if query.include_executions.unwrap_or(false) {
- let pipes_with_execs = mgr.list_pipes_with_executions(5).await;
+ let execution_limit = query.execution_limit.unwrap_or(5).clamp(1, 100);
+ let pipes_with_execs = mgr.list_pipes_with_executions(execution_limit).await;
let total = pipes_with_execs.len();
let data: Vec<Value> = pipes_with_execs
.into_iter()
@@ -267,8 +270,8 @@ pub async fn get_pipe_executions(
Query(query): Query<ExecutionsQuery>,
) -> Json<Value> {
let mgr = pm.lock().await;
- let limit = query.limit.unwrap_or(20).min(100);
- match mgr.get_executions(&id, limit).await {
+ let limit = query.limit.unwrap_or(100).clamp(1, 500);
+ match mgr.get_executions(&id, limit, query.before_id).await {
Ok(executions) => Json(json!({ "data": executions })),
Err(e) => Json(json!({ "error": e.to_string() })),
}
@@ -280,7 +283,7 @@ pub async fn get_pipe_session(
Path((id, exec_id)): Path<(String, i64)>,
) -> (StatusCode, Json<Value>) {
let mgr = pm.lock().await;
- let execs = match mgr.get_executions(&id, 100).await {
+ let execs = match mgr.get_executions(&id, 1, exec_id.checked_add(1)).await {
Ok(e) => e,
Err(e) => {
return (