YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(meetings): stream the summary as it is written (#6247)
Louis Beaumont1985c855ccf2
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.test.ts b/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.test.tsindex 63af7637f..43332a209 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.test.ts+++ b/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.test.ts@@ -13,6 +13,18 @@ const textDelta = (delta: string) => ({assistantMessageEvent: { type: "text_delta", delta },});+/** The shape the screenpipe cloud route actually emits: no deltas, one+ * `text_end` per block carrying the finished string. Captured from+ * `GET /pipes/meeting-summary/executions`. */+const textEnd = (content: string, contentIndex = 0) => ({+ type: "message_update",+ assistantMessageEvent: { type: "text_end", content, contentIndex },+});+const textStart = (contentIndex = 0) => ({+ type: "message_update",+ assistantMessageEvent: { type: "text_start", contentIndex },+});+describe("meeting summary stream", () => {it("streams only markdown after the summary heading", () => {let state = emptyMeetingSummaryStream();@@ -58,4 +70,87 @@ describe("meeting summary stream", () => {expect(state.sealed).toBe(true);expect(state.markdown).toBe("Decision and owner.");});++ // Regression: every recorded production run emitted zero `text_delta` and+ // one-to-four `text_end`, so a delta-only reader streamed nothing and the+ // summary tab showed its skeleton for the whole run.+ it("streams from text_end when the provider emits no deltas", () => {+ let state = emptyMeetingSummaryStream();+ state = advanceMeetingSummaryStream(state, textStart());+ state = advanceMeetingSummaryStream(+ state,+ textEnd("Reading the transcript first."),+ );+ expect(state.markdown).toBe("");++ state = advanceMeetingSummaryStream(state, {+ type: "message_start",+ message: { role: "assistant" },+ });+ state = advanceMeetingSummaryStream(state, textStart());+ state = advanceMeetingSummaryStream(+ state,+ textEnd("## Summary\nThe team **approved** the launch."),+ );++ expect(state.markdown).toBe("The team **approved** the launch.");+ });++ it("keeps multi-block text in contentIndex order", () => {+ let state = emptyMeetingSummaryStream();+ state = advanceMeetingSummaryStream(state, textEnd("## Summary", 0));+ state = advanceMeetingSummaryStream(state, textEnd("Owner: Ana.", 1));++ expect(state.markdown).toBe("Owner: Ana.");+ });++ it("does not double text when a provider sends deltas and text_end", () => {+ let state = emptyMeetingSummaryStream();+ state = advanceMeetingSummaryStream(state, textDelta("## Summary\nShip "));+ state = advanceMeetingSummaryStream(state, textDelta("on Friday."));+ state = advanceMeetingSummaryStream(+ state,+ textEnd("## Summary\nShip on Friday."),+ );++ expect(state.markdown).toBe("Ship on Friday.");+ });++ // Production assistant messages carry `["text", "toolCall"]` together, so the+ // summary and the save `curl` land in one message. Reading the text only at+ // `message_end` therefore revealed it after the save had already run; closing+ // the text block first is what makes it visible while the run is still going.+ it("reveals the summary before the save tool call in the same message", () => {+ let state = emptyMeetingSummaryStream();+ state = advanceMeetingSummaryStream(state, {+ type: "message_start",+ message: { role: "assistant" },+ });+ state = advanceMeetingSummaryStream(state, textStart(0));+ state = advanceMeetingSummaryStream(+ state,+ textEnd("## Summary\nRenewal approved; Ana owns rollout.", 0),+ );++ expect(state.markdown).toBe("Renewal approved; Ana owns rollout.");+ expect(state.sealed).toBe(false);++ state = advanceMeetingSummaryStream(state, {+ type: "message_update",+ assistantMessageEvent: { type: "toolcall_start", contentIndex: 1 },+ });+ expect(state.sealed).toBe(true);+ expect(state.markdown).toBe("Renewal approved; Ana owns rollout.");+ });++ it("ignores tool narration that never reaches a summary heading", () => {+ let state = emptyMeetingSummaryStream();+ state = advanceMeetingSummaryStream(state, textEnd("Good, got data."));+ state = advanceMeetingSummaryStream(+ state,+ textEnd("Only the last ~4 minutes returned. Paginating."),+ );++ expect(state.markdown).toBe("");+ });});diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.ts b/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.tsindex 8f61646fc..045c092b1 100644--- a/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.ts+++ b/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.ts@@ -7,6 +7,8 @@ import type { AgentInnerEvent } from "@/lib/events/types";export interface MeetingSummaryStreamState {rawAssistantText: string;currentMessageText: string;+ /** Text blocks of the in-flight assistant message, keyed by `contentIndex`. */+ blocks: Record<number, string>;markdown: string;sealed: boolean;}@@ -15,11 +17,23 @@ export function emptyMeetingSummaryStream(): MeetingSummaryStreamState {return {rawAssistantText: "",currentMessageText: "",+ blocks: {},markdown: "",sealed: false,};}+/** One assistant message can carry several text blocks; keep them in index+ * order so a later block never overtakes an earlier one. */+function joinBlocks(blocks: Record<number, string>): string {+ return Object.keys(blocks)+ .map(Number)+ .sort((a, b) => a - b)+ .map((index) => blocks[index])+ .filter(Boolean)+ .join("\n");+}+function summaryDraftFromAssistantText(text: string): string | null {const heading = /^#{1,6}[\t ]+summary[\t ]*(?:\r?\n|$)/gim;let match: RegExpExecArray | null;@@ -91,7 +105,7 @@ export function advanceMeetingSummaryStream(}if (type === "message_start" && event.message?.role === "assistant") {- return { ...state, currentMessageText: "" };+ return { ...state, currentMessageText: "", blocks: {} };}const isTextDelta =@@ -100,26 +114,38 @@ export function advanceMeetingSummaryStream(typeof (event.delta ?? assistantEvent?.delta) === "string";if (isTextDelta) {const delta = (event.delta ?? assistantEvent?.delta) as string;- const separator =- !state.currentMessageText &&- state.rawAssistantText &&- !state.rawAssistantText.endsWith("\n")- ? "\n\n"- : "";- const currentMessageText = state.currentMessageText + delta;- const rawAssistantText = state.rawAssistantText + separator + delta;- return {- ...state,- rawAssistantText,- currentMessageText,- markdown:- summaryDraftFromAssistantText(rawAssistantText) ?? state.markdown,+ const index = assistantEvent?.contentIndex ?? 0;+ const blocks = {+ ...state.blocks,+ [index]: (state.blocks[index] ?? "") + delta,};+ return replaceCurrentMessage({ ...state, blocks }, joinBlocks(blocks));+ }++ // Not every provider streams token-by-token. The screenpipe cloud route the+ // summary Pipe actually runs on closes each block with `text_end` carrying+ // the whole string and emits no `text_delta` at all, so a delta-only reader+ // saw the summary as silence and the tab sat on its skeleton for the entire+ // run. `text_end` is authoritative for its block: assigning rather than+ // appending also keeps providers that send both from doubling the text.+ if (+ type === "message_update" &&+ assistantEvent?.type === "text_end" &&+ typeof assistantEvent.content === "string"+ ) {+ const index = assistantEvent.contentIndex ?? 0;+ const blocks = { ...state.blocks, [index]: assistantEvent.content };+ return replaceCurrentMessage({ ...state, blocks }, joinBlocks(blocks));}if (type === "message_end") {const messageText = assistantMessageText(event.message);- if (messageText) return replaceCurrentMessage(state, messageText);+ if (messageText) {+ return replaceCurrentMessage(+ { ...state, blocks: { 0: messageText } },+ messageText,+ );+ }}return state;diff --git a/apps/screenpipe-app-tauri/lib/events/types.ts b/apps/screenpipe-app-tauri/lib/events/types.tsindex 86ea3f49b..8113dbcde 100644--- a/apps/screenpipe-app-tauri/lib/events/types.ts+++ b/apps/screenpipe-app-tauri/lib/events/types.ts@@ -47,6 +47,9 @@ export interface AgentInnerEvent {assistantMessageEvent?: {type?: string;delta?: string;+ /** Full text of the block, carried on `text_end` by providers that do+ * not emit incremental `text_delta` events. */+ content?: string;contentIndex?: number;toolName?: string;partial?: { content?: Array<{ type?: string; name?: string }> };diff --git a/crates/screenpipe-core/assets/pipes/meeting-summary/pipe.md b/crates/screenpipe-core/assets/pipes/meeting-summary/pipe.mdindex 0b91c3b38..a41a0a54a 100644--- a/crates/screenpipe-core/assets/pipes/meeting-summary/pipe.md+++ b/crates/screenpipe-core/assets/pipes/meeting-summary/pipe.md@@ -64,9 +64,11 @@ step 2d — name the speakers from the screen (do this every run, don't ask firsonly rename when the on-screen evidence is unambiguous — never guess from voice alone. note which speakers you renamed (and which you left as-is) in your final message.-step 3 — before saving, write the proposed summary in your response starting on a line with exactly `## Summary`. put only summary content after that heading and use that same markdown in `<YOUR_SUMMARY>`. the meeting UI streams this section while you write it, so do not put planning, tool narration, or save confirmations after the heading.+step 3 — write the summary out as your own message, before you save it. this message must contain no tool call; end the turn after it. start a line with exactly `## Summary` and put the finished summary markdown after that heading. the meeting UI streams this section live while you write it — it is the only way the user sees anything before the run ends — and it is the same markdown you pass as `<YOUR_SUMMARY>` in step 3b.-if your summary is worth saving, append it to the meeting note (and refresh the title in the same call) via:+this step is not optional and it is not the closing report. "meeting 112 was summarized and saved to its record" is a report; it does not satisfy this step. saving a summary you never printed means the user watched a placeholder for the entire run and then got nothing to read, so treat that as a failed run. keep planning, tool narration, and save confirmations out of the section after the heading — those belong in your closing message in step 4.++step 3b — now save it. if your summary is worth saving, append it to the meeting note (and refresh the title in the same call) via:curl -s -X PUT "http://localhost:3030/meetings/<MEETING_ID>" \-H "Authorization: Bearer $SCREENPIPE_LOCAL_API_KEY" \