Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(pipes): show exact run counts with lazy pagination (#5163)

Louis Beaumont8d6839a3cc57
diff --git a/apps/screenpipe-app-tauri/components/chat-sidebar.tsx b/apps/screenpipe-app-tauri/components/chat-sidebar.tsx
index faf27d0da..d83b3a1a9 100644
--- a/apps/screenpipe-app-tauri/components/chat-sidebar.tsx
+++ b/apps/screenpipe-app-tauri/components/chat-sidebar.tsx
@@ -30,7 +30,7 @@
* survives app restart. Delete removes the file.
*/
-import React, { useEffect, useMemo, useState } from "react";
+import React, { useCallback, useEffect, useMemo, useState } from "react";
import { useInterval } from "@/lib/hooks/use-interval";
import { useTauriEvent } from "@/lib/hooks/use-tauri-event";
import {
@@ -103,6 +103,7 @@ import { Button } from "@/components/ui/button";
import { toast } from "@/components/ui/use-toast";
import { normalizeQueueEventPayload } from "@/lib/chat-queue-controls";
import { Skeleton } from "@/components/ui/skeleton";
+import { localFetch } from "@/lib/api";
import {
applySidebarRecentsCap,
buildSidebarRecentsSections,
@@ -401,7 +402,16 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {
);
// Flatten pipe grouped sections into a single items list for rendering
const pipeItems = useMemo(
- () => pipeGroupedSections.flatMap((section) => section.items),
+ () =>
+ pipeGroupedSections.flatMap((section) => section.items).map((item) => {
+ if (item.kind === "group" || !item.session.pipeContext?.pipeName) return item;
+ return {
+ kind: "group" as const,
+ key: `pipe:${item.session.pipeContext.pipeName}`,
+ title: item.session.pipeContext.pipeName,
+ sessions: [item.session],
+ };
+ }),
[pipeGroupedSections],
);
@@ -409,6 +419,38 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {
"screenpipe:pipes-collapsed",
true
);
+ const [pipeExecutionCounts, setPipeExecutionCounts] = useState<Record<string, number>>({});
+ const [pipeExecutionCountsLoaded, setPipeExecutionCountsLoaded] = useState(false);
+ const fetchPipeExecutionCounts = useCallback(async () => {
+ try {
+ const response = await localFetch("/pipes?include_execution_counts=true");
+ if (!response.ok) return;
+ const payload = await response.json();
+ const counts: Record<string, number> = {};
+ for (const pipe of payload.data ?? []) {
+ const name = pipe?.config?.name;
+ if (typeof name === "string" && typeof pipe.execution_count === "number") {
+ counts[name] = pipe.execution_count;
+ }
+ }
+ setPipeExecutionCounts(counts);
+ } catch {
+ // Older engines do not expose exact counts. Keep the recent-chat
+ // fallback rather than making the sidebar unavailable.
+ } finally {
+ setPipeExecutionCountsLoaded(true);
+ }
+ }, []);
+
+ // Counts are intentionally lazy: a collapsed Pipes section performs no
+ // database count query. Refresh while visible so completed runs stay exact.
+ useEffect(() => {
+ if (!pipesCollapsed && pipes.length > 0) void fetchPipeExecutionCounts();
+ }, [pipesCollapsed, pipes.length, fetchPipeExecutionCounts]);
+ useInterval(
+ () => void fetchPipeExecutionCounts(),
+ pipesCollapsed || pipes.length === 0 ? null : 15_000,
+ );
// Auto-expand the pipes section when the current session is a pipe run
// so the highlighted row is visible. Only depends on currentId and the
@@ -905,6 +947,8 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {
<PipeGroupRow
key={item.key}
item={item}
+ executionCount={pipeExecutionCounts[item.title]}
+ executionCountLoading={!pipeExecutionCountsLoaded}
expanded={expandedGroups.has(item.key)}
onToggleExpand={() => toggleGroupExpanded(item.key)}
currentId={currentId}
@@ -1531,6 +1575,8 @@ function RecentsBody({
*/
function PipeGroupRow({
item,
+ executionCount,
+ executionCountLoading = false,
expanded,
onToggleExpand,
currentId,
@@ -1548,6 +1594,8 @@ function PipeGroupRow({
setOpenConversationMenuId,
}: {
item: Extract<SidebarItem, { kind: "group" }>;
+ executionCount?: number;
+ executionCountLoading?: boolean;
expanded: boolean;
onToggleExpand: () => void;
currentId: string | null;
@@ -1578,7 +1626,7 @@ function PipeGroupRow({
<span className="truncate flex-1 text-xs">{item.title}</span>
<span className="inline-flex items-center gap-1.5 shrink-0">
<span className="text-[10px] tabular-nums text-muted-foreground/60">
- {item.sessions.length}
+ {executionCountLoading ? "…" : executionCount ?? item.sessions.length}
</span>
{expanded ? (
<ChevronDown
diff --git a/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx b/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx
index c424c8005..4bcca4016 100644
--- a/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx
@@ -133,7 +133,7 @@ import { useDeviceMonitor } from "@/lib/hooks/use-device-monitor";
import { Monitor, Wifi, WifiOff, ScanSearch } from "lucide-react";
import { requestPipeStop } from "@/lib/pipe-stop";
-const PIPE_EXECUTIONS_PAGE_LIMIT = 100;
+const PIPE_EXECUTIONS_PAGE_LIMIT = 10;
function pipeExecutionsUrl(apiBase: string, pipeName: string, beforeId?: number) {
const params = new URLSearchParams({
@@ -602,6 +602,7 @@ interface PipeStatus {
source_slug?: string;
installed_version?: number;
locally_modified?: boolean;
+ execution_count?: number;
}
interface PipeRunLog {
@@ -1219,7 +1220,7 @@ export function PipesSection() {
const fetchPipes = useCallback(async () => {
try {
setLoadError(null);
- // Load pipes WITH recent executions inline so the list shows the real
+ // Load pipes WITH only their newest execution inline so the list shows the real
// last-run status. Without this the "last run" column always reads
// "never run" for pipes that have actually run (the badge is driven by
// recent_executions). The engine batches this into one fast per-pipe
@@ -1229,8 +1230,8 @@ export function PipesSection() {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 5_000);
const pipesEndpoint = isRemote
- ? `${apiBase}/pipes?include_executions=true`
- : "/pipes?include_executions=true";
+ ? `${apiBase}/pipes?include_executions=true&execution_limit=1&include_execution_counts=true`
+ : "/pipes?include_executions=true&execution_limit=1&include_execution_counts=true";
const res = await localFetch(pipesEndpoint, { signal: controller.signal }).finally(() => clearTimeout(timeout));
if (!res.ok) {
throw new Error(`pipes api returned ${res.status}`);
@@ -1712,7 +1713,12 @@ export function PipesSection() {
const data = await res.json();
const nextExecutions = data.data || [];
setExecutions(nextExecutions);
- setHasMoreExecutions(nextExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);
+ const total = pipes.find((pipe) => pipe.config.name === name)?.execution_count;
+ setHasMoreExecutions(
+ total != null
+ ? nextExecutions.length < total
+ : nextExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT,
+ );
} catch (e) {
// Executions endpoint may not exist on older servers — fall back silently
setExecutions([]);
@@ -1732,14 +1738,18 @@ export function PipesSection() {
const res = await fetch(pipeExecutionsUrl(apiBase, name, oldestId));
const data = await res.json();
const olderExecutions: PipeExecution[] = data.data || [];
- setExecutions((prev) => {
- const seen = new Set(prev.map((exec) => exec.id));
- return [
- ...prev,
- ...olderExecutions.filter((exec) => !seen.has(exec.id)),
- ];
- });
- setHasMoreExecutions(olderExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);
+ const total = pipes.find((pipe) => pipe.config.name === name)?.execution_count;
+ const seen = new Set(executions.map((exec) => exec.id));
+ const next = [
+ ...executions,
+ ...olderExecutions.filter((exec) => !seen.has(exec.id)),
+ ];
+ setExecutions(next);
+ setHasMoreExecutions(
+ total != null
+ ? next.length < total
+ : olderExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT,
+ );
} catch (e) {
console.error("failed to fetch older executions:", e);
} finally {
@@ -2799,7 +2809,9 @@ export function PipesSection() {
config
</TabsTrigger>
<TabsTrigger value="runs" className="rounded-none border-b-2 border-transparent data-[state=active]:border-foreground data-[state=active]:bg-transparent data-[state=active]:shadow-none text-xs uppercase tracking-wider px-3 h-8">
- runs{executions.length > 0 ? ` (${executions.length})` : ""}
+ runs{(pipe.execution_count ?? executions.length) > 0
+ ? ` (${pipe.execution_count ?? executions.length})`
+ : ""}
</TabsTrigger>
<TabsTrigger value="advanced" className="rounded-none border-b-2 border-transparent data-[state=active]:border-foreground data-[state=active]:bg-transparent data-[state=active]:shadow-none text-xs uppercase tracking-wider px-3 h-8">
advanced
diff --git a/crates/screenpipe-core/src/pipes/mod.rs b/crates/screenpipe-core/src/pipes/mod.rs
index d0dc19989..7172c534b 100644
--- a/crates/screenpipe-core/src/pipes/mod.rs
+++ b/crates/screenpipe-core/src/pipes/mod.rs
@@ -1133,6 +1133,9 @@ pub trait PipeStore: Send + Sync {
&self,
limit_per_pipe: i32,
) -> Result<HashMap<String, Vec<PipeExecution>>>;
+
+ /// Get the exact persisted execution count for every pipe.
+ async fn get_all_execution_counts(&self) -> Result<HashMap<String, i64>>;
}
// ---------------------------------------------------------------------------
@@ -2643,6 +2646,15 @@ impl PipeManager {
}
}
+ /// Get exact persisted execution counts for all pipes in one grouped query.
+ pub async fn get_all_execution_counts(&self) -> HashMap<String, i64> {
+ if let Some(ref store) = self.store {
+ store.get_all_execution_counts().await.unwrap_or_default()
+ } else {
+ HashMap::new()
+ }
+ }
+
/// List all pipes with status and recent executions in a single call.
/// Avoids N+1 requests by fetching executions for all pipes at once.
pub async fn list_pipes_with_executions(
diff --git a/crates/screenpipe-engine/src/pipe_store.rs b/crates/screenpipe-engine/src/pipe_store.rs
index af8370032..351042723 100644
--- a/crates/screenpipe-engine/src/pipe_store.rs
+++ b/crates/screenpipe-engine/src/pipe_store.rs
@@ -295,6 +295,18 @@ impl PipeStore for SqlitePipeStore {
}
Ok(map)
}
+
+ async fn get_all_execution_counts(&self) -> Result<std::collections::HashMap<String, i64>> {
+ let rows = sqlx::query_as::<_, (String, i64)>(
+ r#"SELECT pipe_name, COUNT(*)
+ FROM pipe_executions
+ GROUP BY pipe_name"#,
+ )
+ .fetch_all(&self.db.pool)
+ .await?;
+
+ Ok(rows.into_iter().collect())
+ }
}
// ---------------------------------------------------------------------------
@@ -940,4 +952,26 @@ mod tests {
let all = store.get_all_executions(5).await.unwrap();
assert!(all.is_empty());
}
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn test_get_all_execution_counts_are_exact() {
+ let (store, _tmp) = setup_test_store().await;
+ for _ in 0..12 {
+ store
+ .create_execution("pipe-a", "scheduled", "m", None)
+ .await
+ .unwrap();
+ }
+ for _ in 0..3 {
+ store
+ .create_execution("pipe-b", "manual", "m", None)
+ .await
+ .unwrap();
+ }
+
+ let counts = store.get_all_execution_counts().await.unwrap();
+ assert_eq!(counts.len(), 2);
+ assert_eq!(counts["pipe-a"], 12);
+ assert_eq!(counts["pipe-b"], 3);
+ }
}
diff --git a/crates/screenpipe-engine/src/pipes_api.rs b/crates/screenpipe-engine/src/pipes_api.rs
index eb6f26ba2..b5c76b5e4 100644
--- a/crates/screenpipe-engine/src/pipes_api.rs
+++ b/crates/screenpipe-engine/src/pipes_api.rs
@@ -58,6 +58,7 @@ pub struct ExecutionsQuery {
pub struct ListPipesQuery {
pub include_executions: Option<bool>,
pub execution_limit: Option<i32>,
+ pub include_execution_counts: Option<bool>,
}
// ---------------------------------------------------------------------------
@@ -75,6 +76,11 @@ pub async fn list_pipes(
if let Err(e) = mgr.reload_pipes().await {
tracing::warn!("failed to reload pipes from disk: {}", e);
}
+ let execution_counts = if query.include_execution_counts.unwrap_or(false) {
+ mgr.get_all_execution_counts().await
+ } else {
+ HashMap::new()
+ };
if query.include_executions.unwrap_or(false) {
let execution_limit = query.execution_limit.unwrap_or(5).clamp(1, 100);
let pipes_with_execs = mgr.list_pipes_with_executions(execution_limit).await;
@@ -85,6 +91,15 @@ pub async fn list_pipes(
let mut obj = serde_json::to_value(&status).unwrap_or(json!({}));
if let Some(map) = obj.as_object_mut() {
map.insert("recent_executions".to_string(), json!(execs));
+ if query.include_execution_counts.unwrap_or(false) {
+ map.insert(
+ "execution_count".to_string(),
+ json!(execution_counts
+ .get(&status.config.name)
+ .copied()
+ .unwrap_or(0)),
+ );
+ }
}
obj
})
@@ -93,7 +108,27 @@ pub async fn list_pipes(
} else {
let pipes = mgr.list_pipes().await;
let total = pipes.len();
- Json(json!({ "data": pipes, "total": total }))
+ if query.include_execution_counts.unwrap_or(false) {
+ let data: Vec<Value> = pipes
+ .into_iter()
+ .map(|status| {
+ let mut obj = serde_json::to_value(&status).unwrap_or(json!({}));
+ if let Some(map) = obj.as_object_mut() {
+ map.insert(
+ "execution_count".to_string(),
+ json!(execution_counts
+ .get(&status.config.name)
+ .copied()
+ .unwrap_or(0)),
+ );
+ }
+ obj
+ })
+ .collect();
+ Json(json!({ "data": data, "total": total }))
+ } else {
+ Json(json!({ "data": pipes, "total": total }))
+ }
}
}