YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(pipes): show exact run counts with lazy pagination (#5163)
Louis Beaumont8d6839a3cc57
diff --git a/apps/screenpipe-app-tauri/components/chat-sidebar.tsx b/apps/screenpipe-app-tauri/components/chat-sidebar.tsxindex faf27d0da..d83b3a1a9 100644--- a/apps/screenpipe-app-tauri/components/chat-sidebar.tsx+++ b/apps/screenpipe-app-tauri/components/chat-sidebar.tsx@@ -30,7 +30,7 @@* survives app restart. Delete removes the file.*/-import React, { useEffect, useMemo, useState } from "react";+import React, { useCallback, useEffect, useMemo, useState } from "react";import { useInterval } from "@/lib/hooks/use-interval";import { useTauriEvent } from "@/lib/hooks/use-tauri-event";import {@@ -103,6 +103,7 @@ import { Button } from "@/components/ui/button";import { toast } from "@/components/ui/use-toast";import { normalizeQueueEventPayload } from "@/lib/chat-queue-controls";import { Skeleton } from "@/components/ui/skeleton";+import { localFetch } from "@/lib/api";import {applySidebarRecentsCap,buildSidebarRecentsSections,@@ -401,7 +402,16 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {);// Flatten pipe grouped sections into a single items list for renderingconst pipeItems = useMemo(- () => pipeGroupedSections.flatMap((section) => section.items),+ () =>+ pipeGroupedSections.flatMap((section) => section.items).map((item) => {+ if (item.kind === "group" || !item.session.pipeContext?.pipeName) return item;+ return {+ kind: "group" as const,+ key: `pipe:${item.session.pipeContext.pipeName}`,+ title: item.session.pipeContext.pipeName,+ sessions: [item.session],+ };+ }),[pipeGroupedSections],);@@ -409,6 +419,38 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {"screenpipe:pipes-collapsed",true);+ const [pipeExecutionCounts, setPipeExecutionCounts] = useState<Record<string, number>>({});+ const [pipeExecutionCountsLoaded, setPipeExecutionCountsLoaded] = useState(false);+ const fetchPipeExecutionCounts = useCallback(async () => {+ try {+ const response = await localFetch("/pipes?include_execution_counts=true");+ if (!response.ok) return;+ const payload = await response.json();+ const counts: Record<string, number> = {};+ for (const pipe of payload.data ?? []) {+ const name = pipe?.config?.name;+ if (typeof name === "string" && typeof pipe.execution_count === "number") {+ counts[name] = pipe.execution_count;+ }+ }+ setPipeExecutionCounts(counts);+ } catch {+ // Older engines do not expose exact counts. Keep the recent-chat+ // fallback rather than making the sidebar unavailable.+ } finally {+ setPipeExecutionCountsLoaded(true);+ }+ }, []);++ // Counts are intentionally lazy: a collapsed Pipes section performs no+ // database count query. Refresh while visible so completed runs stay exact.+ useEffect(() => {+ if (!pipesCollapsed && pipes.length > 0) void fetchPipeExecutionCounts();+ }, [pipesCollapsed, pipes.length, fetchPipeExecutionCounts]);+ useInterval(+ () => void fetchPipeExecutionCounts(),+ pipesCollapsed || pipes.length === 0 ? null : 15_000,+ );// Auto-expand the pipes section when the current session is a pipe run// so the highlighted row is visible. Only depends on currentId and the@@ -905,6 +947,8 @@ export function ChatSidebar({ className, onViewAll }: ChatSidebarProps) {<PipeGroupRowkey={item.key}item={item}+ executionCount={pipeExecutionCounts[item.title]}+ executionCountLoading={!pipeExecutionCountsLoaded}expanded={expandedGroups.has(item.key)}onToggleExpand={() => toggleGroupExpanded(item.key)}currentId={currentId}@@ -1531,6 +1575,8 @@ function RecentsBody({*/function PipeGroupRow({item,+ executionCount,+ executionCountLoading = false,expanded,onToggleExpand,currentId,@@ -1548,6 +1594,8 @@ function PipeGroupRow({setOpenConversationMenuId,}: {item: Extract<SidebarItem, { kind: "group" }>;+ executionCount?: number;+ executionCountLoading?: boolean;expanded: boolean;onToggleExpand: () => void;currentId: string | null;@@ -1578,7 +1626,7 @@ function PipeGroupRow({<span className="truncate flex-1 text-xs">{item.title}</span><span className="inline-flex items-center gap-1.5 shrink-0"><span className="text-[10px] tabular-nums text-muted-foreground/60">- {item.sessions.length}+ {executionCountLoading ? "…" : executionCount ?? item.sessions.length}</span>{expanded ? (<ChevronDowndiff --git a/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx b/apps/screenpipe-app-tauri/components/settings/pipes-section.tsxindex c424c8005..4bcca4016 100644--- a/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx+++ b/apps/screenpipe-app-tauri/components/settings/pipes-section.tsx@@ -133,7 +133,7 @@ import { useDeviceMonitor } from "@/lib/hooks/use-device-monitor";import { Monitor, Wifi, WifiOff, ScanSearch } from "lucide-react";import { requestPipeStop } from "@/lib/pipe-stop";-const PIPE_EXECUTIONS_PAGE_LIMIT = 100;+const PIPE_EXECUTIONS_PAGE_LIMIT = 10;function pipeExecutionsUrl(apiBase: string, pipeName: string, beforeId?: number) {const params = new URLSearchParams({@@ -602,6 +602,7 @@ interface PipeStatus {source_slug?: string;installed_version?: number;locally_modified?: boolean;+ execution_count?: number;}interface PipeRunLog {@@ -1219,7 +1220,7 @@ export function PipesSection() {const fetchPipes = useCallback(async () => {try {setLoadError(null);- // Load pipes WITH recent executions inline so the list shows the real+ // Load pipes WITH only their newest execution inline so the list shows the real// last-run status. Without this the "last run" column always reads// "never run" for pipes that have actually run (the badge is driven by// recent_executions). The engine batches this into one fast per-pipe@@ -1229,8 +1230,8 @@ export function PipesSection() {const controller = new AbortController();const timeout = setTimeout(() => controller.abort(), 5_000);const pipesEndpoint = isRemote- ? `${apiBase}/pipes?include_executions=true`- : "/pipes?include_executions=true";+ ? `${apiBase}/pipes?include_executions=true&execution_limit=1&include_execution_counts=true`+ : "/pipes?include_executions=true&execution_limit=1&include_execution_counts=true";const res = await localFetch(pipesEndpoint, { signal: controller.signal }).finally(() => clearTimeout(timeout));if (!res.ok) {throw new Error(`pipes api returned ${res.status}`);@@ -1712,7 +1713,12 @@ export function PipesSection() {const data = await res.json();const nextExecutions = data.data || [];setExecutions(nextExecutions);- setHasMoreExecutions(nextExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);+ const total = pipes.find((pipe) => pipe.config.name === name)?.execution_count;+ setHasMoreExecutions(+ total != null+ ? nextExecutions.length < total+ : nextExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT,+ );} catch (e) {// Executions endpoint may not exist on older servers — fall back silentlysetExecutions([]);@@ -1732,14 +1738,18 @@ export function PipesSection() {const res = await fetch(pipeExecutionsUrl(apiBase, name, oldestId));const data = await res.json();const olderExecutions: PipeExecution[] = data.data || [];- setExecutions((prev) => {- const seen = new Set(prev.map((exec) => exec.id));- return [- ...prev,- ...olderExecutions.filter((exec) => !seen.has(exec.id)),- ];- });- setHasMoreExecutions(olderExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT);+ const total = pipes.find((pipe) => pipe.config.name === name)?.execution_count;+ const seen = new Set(executions.map((exec) => exec.id));+ const next = [+ ...executions,+ ...olderExecutions.filter((exec) => !seen.has(exec.id)),+ ];+ setExecutions(next);+ setHasMoreExecutions(+ total != null+ ? next.length < total+ : olderExecutions.length === PIPE_EXECUTIONS_PAGE_LIMIT,+ );} catch (e) {console.error("failed to fetch older executions:", e);} finally {@@ -2799,7 +2809,9 @@ export function PipesSection() {config</TabsTrigger><TabsTrigger value="runs" className="rounded-none border-b-2 border-transparent data-[state=active]:border-foreground data-[state=active]:bg-transparent data-[state=active]:shadow-none text-xs uppercase tracking-wider px-3 h-8">- runs{executions.length > 0 ? ` (${executions.length})` : ""}+ runs{(pipe.execution_count ?? executions.length) > 0+ ? ` (${pipe.execution_count ?? executions.length})`+ : ""}</TabsTrigger><TabsTrigger value="advanced" className="rounded-none border-b-2 border-transparent data-[state=active]:border-foreground data-[state=active]:bg-transparent data-[state=active]:shadow-none text-xs uppercase tracking-wider px-3 h-8">advanceddiff --git a/crates/screenpipe-core/src/pipes/mod.rs b/crates/screenpipe-core/src/pipes/mod.rsindex d0dc19989..7172c534b 100644--- a/crates/screenpipe-core/src/pipes/mod.rs+++ b/crates/screenpipe-core/src/pipes/mod.rs@@ -1133,6 +1133,9 @@ pub trait PipeStore: Send + Sync {&self,limit_per_pipe: i32,) -> Result<HashMap<String, Vec<PipeExecution>>>;++ /// Get the exact persisted execution count for every pipe.+ async fn get_all_execution_counts(&self) -> Result<HashMap<String, i64>>;}// ---------------------------------------------------------------------------@@ -2643,6 +2646,15 @@ impl PipeManager {}}+ /// Get exact persisted execution counts for all pipes in one grouped query.+ pub async fn get_all_execution_counts(&self) -> HashMap<String, i64> {+ if let Some(ref store) = self.store {+ store.get_all_execution_counts().await.unwrap_or_default()+ } else {+ HashMap::new()+ }+ }+/// List all pipes with status and recent executions in a single call./// Avoids N+1 requests by fetching executions for all pipes at once.pub async fn list_pipes_with_executions(diff --git a/crates/screenpipe-engine/src/pipe_store.rs b/crates/screenpipe-engine/src/pipe_store.rsindex af8370032..351042723 100644--- a/crates/screenpipe-engine/src/pipe_store.rs+++ b/crates/screenpipe-engine/src/pipe_store.rs@@ -295,6 +295,18 @@ impl PipeStore for SqlitePipeStore {}Ok(map)}++ async fn get_all_execution_counts(&self) -> Result<std::collections::HashMap<String, i64>> {+ let rows = sqlx::query_as::<_, (String, i64)>(+ r#"SELECT pipe_name, COUNT(*)+ FROM pipe_executions+ GROUP BY pipe_name"#,+ )+ .fetch_all(&self.db.pool)+ .await?;++ Ok(rows.into_iter().collect())+ }}// ---------------------------------------------------------------------------@@ -940,4 +952,26 @@ mod tests {let all = store.get_all_executions(5).await.unwrap();assert!(all.is_empty());}++ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn test_get_all_execution_counts_are_exact() {+ let (store, _tmp) = setup_test_store().await;+ for _ in 0..12 {+ store+ .create_execution("pipe-a", "scheduled", "m", None)+ .await+ .unwrap();+ }+ for _ in 0..3 {+ store+ .create_execution("pipe-b", "manual", "m", None)+ .await+ .unwrap();+ }++ let counts = store.get_all_execution_counts().await.unwrap();+ assert_eq!(counts.len(), 2);+ assert_eq!(counts["pipe-a"], 12);+ assert_eq!(counts["pipe-b"], 3);+ }}diff --git a/crates/screenpipe-engine/src/pipes_api.rs b/crates/screenpipe-engine/src/pipes_api.rsindex eb6f26ba2..b5c76b5e4 100644--- a/crates/screenpipe-engine/src/pipes_api.rs+++ b/crates/screenpipe-engine/src/pipes_api.rs@@ -58,6 +58,7 @@ pub struct ExecutionsQuery {pub struct ListPipesQuery {pub include_executions: Option<bool>,pub execution_limit: Option<i32>,+ pub include_execution_counts: Option<bool>,}// ---------------------------------------------------------------------------@@ -75,6 +76,11 @@ pub async fn list_pipes(if let Err(e) = mgr.reload_pipes().await {tracing::warn!("failed to reload pipes from disk: {}", e);}+ let execution_counts = if query.include_execution_counts.unwrap_or(false) {+ mgr.get_all_execution_counts().await+ } else {+ HashMap::new()+ };if query.include_executions.unwrap_or(false) {let execution_limit = query.execution_limit.unwrap_or(5).clamp(1, 100);let pipes_with_execs = mgr.list_pipes_with_executions(execution_limit).await;@@ -85,6 +91,15 @@ pub async fn list_pipes(let mut obj = serde_json::to_value(&status).unwrap_or(json!({}));if let Some(map) = obj.as_object_mut() {map.insert("recent_executions".to_string(), json!(execs));+ if query.include_execution_counts.unwrap_or(false) {+ map.insert(+ "execution_count".to_string(),+ json!(execution_counts+ .get(&status.config.name)+ .copied()+ .unwrap_or(0)),+ );+ }}obj})@@ -93,7 +108,27 @@ pub async fn list_pipes(} else {let pipes = mgr.list_pipes().await;let total = pipes.len();- Json(json!({ "data": pipes, "total": total }))+ if query.include_execution_counts.unwrap_or(false) {+ let data: Vec<Value> = pipes+ .into_iter()+ .map(|status| {+ let mut obj = serde_json::to_value(&status).unwrap_or(json!({}));+ if let Some(map) = obj.as_object_mut() {+ map.insert(+ "execution_count".to_string(),+ json!(execution_counts+ .get(&status.config.name)+ .copied()+ .unwrap_or(0)),+ );+ }+ obj+ })+ .collect();+ Json(json!({ "data": data, "total": total }))+ } else {+ Json(json!({ "data": pipes, "total": total }))+ }}}