YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(chat): preserve active turn feedback across windows (#5657)
Louis Beaumontd667a4dd32c2
diff --git a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-foreground-events.ts b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-foreground-events.tsindex aa59e73c0..dfb741bbe 100644--- a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-foreground-events.ts+++ b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-foreground-events.ts@@ -16,6 +16,7 @@ import { buildDailyLimitMessage, buildRateLimitMessage, classifyQuotaError, parsimport { buildInvalidatedAuthTokenMessage, isInvalidatedAuthTokenError } from "@/lib/chat/auth-errors";import { buildNoResponseMessage, buildProviderErrorMessage } from "@/lib/chat/provider-errors";import { chatTelemetryContextForResponse } from "@/lib/chat/response-feedback";+import { optimisticAssistantForUserEcho } from "@/lib/chat/cross-window-transcript-sync";import { qualifiedValue } from "@/lib/analytics/qualified-value";import { registerPiLogListener } from "@/components/chat/standalone/hooks/pi-log-listener";import { registerPiReauthListener } from "@/components/chat/standalone/hooks/pi-reauth-listener";@@ -458,6 +459,29 @@ export function usePiForegroundEvents({// point `sendPiMessage` has just created an empty placeholder and// there's nothing streamed yet (clearing would orphan the// placeholder and re-create a duplicate on the first delta).+ const rawText = textFromMessageContent(data.message?.content);+ const text = extractInjectedUserText(rawText) ?? rawText;+ const sidForStartedUser = piSessionIdRef.current;++ // A sibling WebView can receive this Pi echo before React commits+ // the saved transcript and active assistant ref locally. The+ // chat-store is updated synchronously by cross-window hydration, so+ // consult both sources before interpreting the echo as a queued turn.+ // Otherwise one card click persists the prompt + placeholder twice.+ if (!piMessageIdRef.current) {+ const storedMessages = sidForStartedUser+ ? useChatStore.getState().sessions[sidForStartedUser]?.messages+ : undefined;+ const optimisticAssistant =+ optimisticAssistantForUserEcho(messagesRef.current, text) ??+ optimisticAssistantForUserEcho((storedMessages ?? []) as Message[], text);+ if (optimisticAssistant) {+ piMessageIdRef.current = optimisticAssistant.assistantMessageId;+ piStreamingTextRef.current = optimisticAssistant.streamingText;+ piContentBlocksRef.current = optimisticAssistant.contentBlocks;+ }+ }+const hasStreamedContent =piStreamingTextRef.current.length > 0 ||piContentBlocksRef.current.length > 0;@@ -470,8 +494,6 @@ export function usePiForegroundEvents({// processing the followUp turn.}- const rawText = textFromMessageContent(data.message?.content);- const text = extractInjectedUserText(rawText) ?? rawText;const eventImages = imageDataUrlsFromPiContent(data.message?.content);const pendingOptimisticSteer = optimisticSteerRef.current;const isPendingOptimisticSteerEcho = Boolean(@@ -482,7 +504,6 @@ export function usePiForegroundEvents({const preMatchedTurnIntent = findTurnIntentForUserStart(piSessionIdRef.current, text, pendingNextPiUserDisplayRef.current);if (!piMessageIdRef.current || isPendingOptimisticSteerEcho || preMatchedTurnIntent?.kind === "steer") {- const sidForStartedUser = piSessionIdRef.current;const pendingDisplay = pendingNextPiUserDisplayRef.current &&(!text || turnIntentTextValuesMatch(pendingNextPiUserDisplayRef.current.preview, text))? pendingNextPiUserDisplayRef.current@@ -565,6 +586,7 @@ export function usePiForegroundEvents({idOverride: piSessionIdRef.current,refreshHistory: false,syncActiveConversation: false,+ turnState: { isLoading: true, isStreaming: true },});}diff --git a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-steering-transport.ts b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-steering-transport.tsindex 963b67320..2023ae703 100644--- a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-steering-transport.ts+++ b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-steering-transport.ts@@ -102,6 +102,7 @@ export function usePiSteeringTransport(void saveConversation(nextRows, {refreshHistory: false,syncActiveConversation: false,+ turnState: { isLoading: true, isStreaming: true },});const sidNow = piSessionIdRef.current;if (sidNow) {@@ -214,6 +215,7 @@ export function usePiSteeringTransport(void saveConversation(nextRowsAfterLabels, {refreshHistory: false,syncActiveConversation: false,+ turnState: { isLoading: true, isStreaming: true },});useChatStore.getState().actions.setMessages(sessionId, nextRowsAfterLabels as any);}@@ -251,6 +253,7 @@ export function usePiSteeringTransport(void saveConversation(nextRowsAfterAssistant, {refreshHistory: false,syncActiveConversation: false,+ turnState: { isLoading: true, isStreaming: true },});useChatStore.getState().actions.setMessages(sessionId, nextRowsAfterAssistant as any);}@@ -416,6 +419,7 @@ export function usePiSteeringTransport(void saveConversation(nextRowsAfterOptimisticAppend, {refreshHistory: false,syncActiveConversation: false,+ turnState: { isLoading: true, isStreaming: true },});}const sidNow = piSessionIdRef.current;@@ -585,6 +589,7 @@ export function usePiSteeringTransport(void saveConversation(nextRowsAfterQueuedSteer, {refreshHistory: false,syncActiveConversation: false,+ turnState: { isLoading: true, isStreaming: true },});const sidNow = piSessionIdRef.current;if (sidNow) {diff --git a/apps/screenpipe-app-tauri/components/hooks/use-chat-conversations.ts b/apps/screenpipe-app-tauri/components/hooks/use-chat-conversations.tsindex bab765332..d76593d67 100644--- a/apps/screenpipe-app-tauri/components/hooks/use-chat-conversations.ts+++ b/apps/screenpipe-app-tauri/components/hooks/use-chat-conversations.ts@@ -44,6 +44,7 @@ import {} from "@/lib/chat-storage";import type { ContentBlock, Message } from "@/lib/chat/types";import {+ savedTurnEventState,shouldAdoptPersistedTranscript,synchronizedActiveTurn,toRuntimeMessages,@@ -426,11 +427,19 @@ export function useChatConversations(opts: UseChatConversationsOpts) {titleSource?: "fallback" | "ai" | "user";updatedAt?: number;turnState?: { isLoading: boolean; isStreaming: boolean };+ activeAssistantMessageId?: string;}>("chat-conversation-saved",async (event) => {if (cancelled) return;- const { id, title, titleSource, updatedAt, turnState } = event.payload ?? {};+ const {+ id,+ title,+ titleSource,+ updatedAt,+ turnState,+ activeAssistantMessageId,+ } = event.payload ?? {};if (!id) return;// Each WebView owns separate React/Zustand state. A sibling can save@@ -439,6 +448,19 @@ export function useChatConversations(opts: UseChatConversationsOpts) {// strictly more complete so a stale save never rolls back a live// foreground stream.if (id === conversationId || id === piSessionIdRef.current) {+ // Publish the stable assistant identity before any disk/title+ // awaits below. Pi can echo the user prompt within milliseconds;+ // without this synchronous handoff a sibling treats that echo as+ // a queued turn and persists the prompt + placeholder twice.+ if (+ activeAssistantMessageId &&+ (turnState?.isLoading || turnState?.isStreaming) &&+ !piMessageIdRef.current+ ) {+ piMessageIdRef.current = activeAssistantMessageId;+ piStreamingTextRef.current = "";+ piContentBlocksRef.current = [];+ }if (typeof updatedAt === "number") {latestSavedEventAtRef.current.set(id,@@ -917,12 +939,13 @@ export function useChatConversations(opts: UseChatConversationsOpts) {upsertFileConversationMeta(conversation);}try {+ const emittedTurnState = savedTurnEventState(msgs, options.turnState);await emit("chat-conversation-saved", {id: conversation.id,title: conversation.title,titleSource: conversation.titleSource,updatedAt: conversation.updatedAt,- turnState: options.turnState ?? { isLoading, isStreaming },+ ...emittedTurnState,});} catch {// ignore broadcast failures; local save already succeeded@@ -995,7 +1018,9 @@ export function useChatConversations(opts: UseChatConversationsOpts) {// If the user has typed follow-up messages, some won't have pipe- IDs → save.const allPipe = messages.every((m) => m.id?.startsWith("pipe-"));if (!allPipe) {- saveConversation(messages);+ saveConversation(messages, {+ turnState: { isLoading: false, isStreaming: false },+ });// Reveal this session in the sidebar — the assistant has replied,// so it's no longer an empty draft.void (async () => {@@ -1053,6 +1078,7 @@ export function useChatConversations(opts: UseChatConversationsOpts) {saveConversation(messages, {refreshHistory: false,syncActiveConversation: false,+ turnState: { isLoading, isStreaming },});}, 1500);diff --git a/apps/screenpipe-app-tauri/e2e/specs/chat-automation-card-duplicate.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/chat-automation-card-duplicate.spec.tsindex 49d472214..4ffd2d65f 100644--- a/apps/screenpipe-app-tauri/e2e/specs/chat-automation-card-duplicate.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/chat-automation-card-duplicate.spec.ts@@ -41,7 +41,12 @@ const CARD_DISPLAY_LABELS: Record<string, string> = {"missed-todos": "✅ Missed To-Dos",};-function chatFilesForDisplayLabel(displayLabel: string): string[] {+interface MatchingChat {+ name: string;+ matchingUserMessages: number;+}++function chatsForDisplayLabel(displayLabel: string): MatchingChat[] {let names: string[];try {names = readdirSync(CHATS_DIR);@@ -49,21 +54,26 @@ function chatFilesForDisplayLabel(displayLabel: string): string[] {return [];}- return names.filter((name) => {- if (!name.endsWith(".json")) return false;+ return names.flatMap((name) => {+ if (!name.endsWith(".json")) return [];try {const conversation = JSON.parse(readFileSync(join(CHATS_DIR, name), "utf8")) as {messages?: Array<{ role?: string; displayContent?: string }>;};- return (conversation.messages ?? []).some(+ const matchingUserMessages = (conversation.messages ?? []).filter((message) => message.role === "user" && message.displayContent === displayLabel,- );+ ).length;+ return matchingUserMessages > 0 ? [{ name, matchingUserMessages }] : [];} catch {- return false;+ return [];}});}+function chatFilesForDisplayLabel(displayLabel: string): string[] {+ return chatsForDisplayLabel(displayLabel).map((chat) => chat.name);+}+function cleanupCardChats(displayLabel: string): void {for (const name of chatFilesForDisplayLabel(displayLabel)) {try {@@ -159,6 +169,14 @@ describe("Automation cards create exactly one chat each (#4719)", function () {);}expect(matches).toHaveLength(1);+ const [persisted] = chatsForDisplayLabel(displayLabel);+ if (persisted?.matchingUserMessages !== 1) {+ throw new Error(+ `BUG REPRODUCED: '${slug}' persisted the same user turn ` ++ `${persisted?.matchingUserMessages ?? 0} times in ${persisted?.name ?? "unknown file"}`,+ );+ }+ expect(persisted.matchingUserMessages).toBe(1);});}});diff --git a/apps/screenpipe-app-tauri/e2e/specs/chat-cross-window-transcript-sync.spec.ts b/apps/screenpipe-app-tauri/e2e/specs/chat-cross-window-transcript-sync.spec.tsindex 4e28359bc..bc9e34635 100644--- a/apps/screenpipe-app-tauri/e2e/specs/chat-cross-window-transcript-sync.spec.ts+++ b/apps/screenpipe-app-tauri/e2e/specs/chat-cross-window-transcript-sync.spec.ts@@ -28,7 +28,7 @@ const ASSISTANT_MARKER = "E2E-CROSS-WINDOW-ANSWER-4P8V2N";function writeConversation(updatedAt: number,- state: "empty" | "active" | "complete" = "empty",+ state: "empty" | "active" | "active-tool" | "complete" = "empty",): void {mkdirSync(CHATS_DIR, { recursive: true });writeFileSync(@@ -56,7 +56,7 @@ function writeConversation(timestamp: updatedAt,},]- : state === "active"+ : state === "active" || state === "active-tool"? [{id: "cross-window-user",@@ -67,7 +67,25 @@ function writeConversation({id: "cross-window-assistant",role: "assistant",- content: "Processing...",+ content: state === "active-tool" ? "" : "Processing...",+ ...(state === "active-tool"+ ? {+ contentBlocks: [+ {+ type: "tool",+ toolCall: {+ id: "cross-window-tool",+ toolName: "read",+ args: { path: "SKILL.md" },+ result: "instructions loaded",+ isRunning: false,+ startedAtMs: updatedAt - 2_000,+ endedAtMs: updatedAt - 1_000,+ },+ },+ ],+ }+ : {}),timestamp: updatedAt,},]@@ -198,6 +216,15 @@ async function expectSynchronizedActiveTurn(): Promise<void> {});}+async function expectActiveToolState(): Promise<void> {+ const summary = await $('[data-testid="tool-activity-summary"]');+ await summary.waitForDisplayed({ timeout: t(10_000) });+ expect((await summary.getText()).toLowerCase()).not.toContain("done");+ const indicator = await summary.$('[data-testid="tool-activity-running-indicator"]');+ await indicator.waitForDisplayed({ timeout: t(10_000) });+ expect(await $('[aria-label="stop reply"]').isDisplayed()).toBe(true);+}+describe("Cross-window chat transcript sync", function () {this.timeout(150_000);@@ -284,6 +311,37 @@ describe("Cross-window chat transcript sync", function () {await browser.switchToWindow("chat");await expectSynchronizedActiveTurn();+ const toolAt = nextFixtureUpdatedAt();+ writeConversation(toolAt, "active-tool");+ await emitTauri("chat-conversation-saved", {+ id: CHAT_ID,+ title: "cross-window transcript sync",+ titleSource: "fallback",+ updatedAt: toolAt,+ turnState: { isLoading: true, isStreaming: true },+ });+ await expectActiveToolState();++ await browser.switchToWindow("home");+ await expectActiveToolState();++ // A sibling may persist title/sidebar/transcript metadata while another+ // WebView owns the live Pi turn. Metadata-only saves must not infer idle+ // from that sibling's local React flags or the running tool receipt flips+ // to "done" before agent_end.+ const metadataOnlyAt = nextFixtureUpdatedAt();+ writeConversation(metadataOnlyAt, "active-tool");+ await emitTauri("chat-conversation-saved", {+ id: CHAT_ID,+ title: "cross-window transcript sync",+ titleSource: "fallback",+ updatedAt: metadataOnlyAt,+ });+ await expectActiveToolState();++ await browser.switchToWindow("chat");+ await expectActiveToolState();+const activeScreenshot = await saveScreenshot("chat-cross-window-active-turn");expect(existsSync(activeScreenshot)).toBe(true);diff --git a/apps/screenpipe-app-tauri/lib/chat/__tests__/cross-window-transcript-sync.test.ts b/apps/screenpipe-app-tauri/lib/chat/__tests__/cross-window-transcript-sync.test.tsindex a8fb07280..4afbd809d 100644--- a/apps/screenpipe-app-tauri/lib/chat/__tests__/cross-window-transcript-sync.test.ts+++ b/apps/screenpipe-app-tauri/lib/chat/__tests__/cross-window-transcript-sync.test.ts@@ -5,6 +5,8 @@import { describe, expect, it } from "vitest";import type { Message } from "@/lib/chat/types";import {+ optimisticAssistantForUserEcho,+ savedTurnEventState,shouldAdoptPersistedTranscript,synchronizedActiveTurn,toRuntimeMessages,@@ -17,6 +19,45 @@ const user: Message = {timestamp: 1,};+describe("saved turn event state", () => {+ const activeMessages: Message[] = [+ { id: "user-1", role: "user", content: "recap my day", timestamp: 1 },+ {+ id: "assistant-1",+ role: "assistant",+ content: "Processing...",+ timestamp: 2,+ },+ ];++ it("does not infer idle from a save that does not own the turn lifecycle", () => {+ expect(savedTurnEventState(activeMessages)).toEqual({});+ });++ it("publishes the stable assistant id for an explicit active save", () => {+ expect(+ savedTurnEventState(activeMessages, {+ isLoading: true,+ isStreaming: true,+ }),+ ).toEqual({+ turnState: { isLoading: true, isStreaming: true },+ activeAssistantMessageId: "assistant-1",+ });+ });++ it("publishes completion without claiming an active assistant", () => {+ expect(+ savedTurnEventState(activeMessages, {+ isLoading: false,+ isStreaming: false,+ }),+ ).toEqual({+ turnState: { isLoading: false, isStreaming: false },+ });+ });+});+describe("cross-window transcript sync", () => {it("hydrates a blank WebView from a completed disk transcript", () => {expect(@@ -104,4 +145,51 @@ describe("cross-window transcript sync", () => {),).toBeNull();});++ it("recovers the optimistic assistant when a sibling receives the Pi user echo", () => {+ expect(+ optimisticAssistantForUserEcho(+ [+ user,+ {+ id: "assistant-stable",+ role: "assistant",+ content: "Processing...",+ timestamp: 2,+ },+ ],+ "prepare the call",+ ),+ ).toEqual({+ assistantMessageId: "assistant-stable",+ streamingText: "",+ contentBlocks: [],+ });+ });++ it("does not collapse a different or already-settled user turn", () => {+ expect(+ optimisticAssistantForUserEcho(+ [+ user,+ { id: "assistant-1", role: "assistant", content: "ready", timestamp: 2 },+ ],+ "prepare the call",+ ),+ ).toBeNull();+ expect(+ optimisticAssistantForUserEcho(+ [+ user,+ {+ id: "assistant-1",+ role: "assistant",+ content: "Processing...",+ timestamp: 2,+ },+ ],+ "prepare a different call",+ ),+ ).toBeNull();+ });});diff --git a/apps/screenpipe-app-tauri/lib/chat/cross-window-transcript-sync.ts b/apps/screenpipe-app-tauri/lib/chat/cross-window-transcript-sync.tsindex 658699fde..d26bae124 100644--- a/apps/screenpipe-app-tauri/lib/chat/cross-window-transcript-sync.ts+++ b/apps/screenpipe-app-tauri/lib/chat/cross-window-transcript-sync.ts@@ -12,6 +12,11 @@ export interface SynchronizedActiveTurn {contentBlocks: ContentBlock[];}+export interface SavedTurnState {+ isLoading: boolean;+ isStreaming: boolean;+}+function blockProgress(block: ContentBlock): number {switch (block.type) {case "text":@@ -41,6 +46,63 @@ function isProcessingPlaceholder(message: Message): boolean {return message.role === "assistant" && message.content.trim() === "Processing...";}+/**+ * Recover the optimistic assistant row when Pi echoes the user prompt before+ * this WebView has finished hydrating the sibling's active-turn refs.+ *+ * The optimistic transcript is already durable at this point. Treating the+ * echo as a new queued turn appends the same user prompt and a second+ * "Processing..." row to that transcript. Match only the exact trailing+ * user/placeholder pair so a later intentional repeat remains a new turn.+ */+export function optimisticAssistantForUserEcho(+ messages: Message[],+ echoedUserText: string,+): SynchronizedActiveTurn | null {+ const normalizedEcho = echoedUserText.trim();+ if (!normalizedEcho || messages.length < 2) return null;++ const user = messages[messages.length - 2];+ const assistant = messages[messages.length - 1];+ if (+ user.role !== "user" ||+ user.content.trim() !== normalizedEcho ||+ !isProcessingPlaceholder(assistant)+ ) {+ return null;+ }++ return {+ assistantMessageId: assistant.id,+ streamingText: "",+ contentBlocks: assistant.contentBlocks ? [...assistant.contentBlocks] : [],+ };+}++/**+ * Build the active-turn fields for a cross-window save event.+ *+ * A WebView that merely persists metadata or a transcript snapshot does not+ * own Pi's lifecycle and must not infer an idle state from its local React+ * flags. Only callers that explicitly observed a turn transition may publish+ * turnState. This prevents a sibling save from changing a live "working"+ * receipt to "done" before agent_end.+ */+export function savedTurnEventState(+ messages: Message[],+ turnState?: SavedTurnState,+): { turnState?: SavedTurnState; activeAssistantMessageId?: string } {+ if (!turnState) return {};++ const activeTurn = synchronizedActiveTurn(messages, turnState);+ return {+ turnState,+ ...(activeTurn+ ? { activeAssistantMessageId: activeTurn.assistantMessageId }+ : {}),+ };+}+/*** A save event is broadcast to every WebView. Adopt its disk transcript only* when it advances the local one; a foreground stream can be newer than the