Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(meetings): stream the summary as it is written (#6247)

Louis Beaumont1985c855ccf2
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.test.ts b/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.test.ts
index 63af7637f..43332a209 100644
--- a/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.test.ts
+++ b/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.test.ts
@@ -13,6 +13,18 @@ const textDelta = (delta: string) => ({
assistantMessageEvent: { type: "text_delta", delta },
});
+/** The shape the screenpipe cloud route actually emits: no deltas, one
+ * `text_end` per block carrying the finished string. Captured from
+ * `GET /pipes/meeting-summary/executions`. */
+const textEnd = (content: string, contentIndex = 0) => ({
+ type: "message_update",
+ assistantMessageEvent: { type: "text_end", content, contentIndex },
+});
+const textStart = (contentIndex = 0) => ({
+ type: "message_update",
+ assistantMessageEvent: { type: "text_start", contentIndex },
+});
+
describe("meeting summary stream", () => {
it("streams only markdown after the summary heading", () => {
let state = emptyMeetingSummaryStream();
@@ -58,4 +70,87 @@ describe("meeting summary stream", () => {
expect(state.sealed).toBe(true);
expect(state.markdown).toBe("Decision and owner.");
});
+
+ // Regression: every recorded production run emitted zero `text_delta` and
+ // one-to-four `text_end`, so a delta-only reader streamed nothing and the
+ // summary tab showed its skeleton for the whole run.
+ it("streams from text_end when the provider emits no deltas", () => {
+ let state = emptyMeetingSummaryStream();
+ state = advanceMeetingSummaryStream(state, textStart());
+ state = advanceMeetingSummaryStream(
+ state,
+ textEnd("Reading the transcript first."),
+ );
+ expect(state.markdown).toBe("");
+
+ state = advanceMeetingSummaryStream(state, {
+ type: "message_start",
+ message: { role: "assistant" },
+ });
+ state = advanceMeetingSummaryStream(state, textStart());
+ state = advanceMeetingSummaryStream(
+ state,
+ textEnd("## Summary\nThe team **approved** the launch."),
+ );
+
+ expect(state.markdown).toBe("The team **approved** the launch.");
+ });
+
+ it("keeps multi-block text in contentIndex order", () => {
+ let state = emptyMeetingSummaryStream();
+ state = advanceMeetingSummaryStream(state, textEnd("## Summary", 0));
+ state = advanceMeetingSummaryStream(state, textEnd("Owner: Ana.", 1));
+
+ expect(state.markdown).toBe("Owner: Ana.");
+ });
+
+ it("does not double text when a provider sends deltas and text_end", () => {
+ let state = emptyMeetingSummaryStream();
+ state = advanceMeetingSummaryStream(state, textDelta("## Summary\nShip "));
+ state = advanceMeetingSummaryStream(state, textDelta("on Friday."));
+ state = advanceMeetingSummaryStream(
+ state,
+ textEnd("## Summary\nShip on Friday."),
+ );
+
+ expect(state.markdown).toBe("Ship on Friday.");
+ });
+
+ // Production assistant messages carry `["text", "toolCall"]` together, so the
+ // summary and the save `curl` land in one message. Reading the text only at
+ // `message_end` therefore revealed it after the save had already run; closing
+ // the text block first is what makes it visible while the run is still going.
+ it("reveals the summary before the save tool call in the same message", () => {
+ let state = emptyMeetingSummaryStream();
+ state = advanceMeetingSummaryStream(state, {
+ type: "message_start",
+ message: { role: "assistant" },
+ });
+ state = advanceMeetingSummaryStream(state, textStart(0));
+ state = advanceMeetingSummaryStream(
+ state,
+ textEnd("## Summary\nRenewal approved; Ana owns rollout.", 0),
+ );
+
+ expect(state.markdown).toBe("Renewal approved; Ana owns rollout.");
+ expect(state.sealed).toBe(false);
+
+ state = advanceMeetingSummaryStream(state, {
+ type: "message_update",
+ assistantMessageEvent: { type: "toolcall_start", contentIndex: 1 },
+ });
+ expect(state.sealed).toBe(true);
+ expect(state.markdown).toBe("Renewal approved; Ana owns rollout.");
+ });
+
+ it("ignores tool narration that never reaches a summary heading", () => {
+ let state = emptyMeetingSummaryStream();
+ state = advanceMeetingSummaryStream(state, textEnd("Good, got data."));
+ state = advanceMeetingSummaryStream(
+ state,
+ textEnd("Only the last ~4 minutes returned. Paginating."),
+ );
+
+ expect(state.markdown).toBe("");
+ });
});
diff --git a/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.ts b/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.ts
index 8f61646fc..045c092b1 100644
--- a/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.ts
+++ b/apps/screenpipe-app-tauri/components/meeting-notes/meeting-summary-stream.ts
@@ -7,6 +7,8 @@ import type { AgentInnerEvent } from "@/lib/events/types";
export interface MeetingSummaryStreamState {
rawAssistantText: string;
currentMessageText: string;
+ /** Text blocks of the in-flight assistant message, keyed by `contentIndex`. */
+ blocks: Record<number, string>;
markdown: string;
sealed: boolean;
}
@@ -15,11 +17,23 @@ export function emptyMeetingSummaryStream(): MeetingSummaryStreamState {
return {
rawAssistantText: "",
currentMessageText: "",
+ blocks: {},
markdown: "",
sealed: false,
};
}
+/** One assistant message can carry several text blocks; keep them in index
+ * order so a later block never overtakes an earlier one. */
+function joinBlocks(blocks: Record<number, string>): string {
+ return Object.keys(blocks)
+ .map(Number)
+ .sort((a, b) => a - b)
+ .map((index) => blocks[index])
+ .filter(Boolean)
+ .join("\n");
+}
+
function summaryDraftFromAssistantText(text: string): string | null {
const heading = /^#{1,6}[\t ]+summary[\t ]*(?:\r?\n|$)/gim;
let match: RegExpExecArray | null;
@@ -91,7 +105,7 @@ export function advanceMeetingSummaryStream(
}
if (type === "message_start" && event.message?.role === "assistant") {
- return { ...state, currentMessageText: "" };
+ return { ...state, currentMessageText: "", blocks: {} };
}
const isTextDelta =
@@ -100,26 +114,38 @@ export function advanceMeetingSummaryStream(
typeof (event.delta ?? assistantEvent?.delta) === "string";
if (isTextDelta) {
const delta = (event.delta ?? assistantEvent?.delta) as string;
- const separator =
- !state.currentMessageText &&
- state.rawAssistantText &&
- !state.rawAssistantText.endsWith("\n")
- ? "\n\n"
- : "";
- const currentMessageText = state.currentMessageText + delta;
- const rawAssistantText = state.rawAssistantText + separator + delta;
- return {
- ...state,
- rawAssistantText,
- currentMessageText,
- markdown:
- summaryDraftFromAssistantText(rawAssistantText) ?? state.markdown,
+ const index = assistantEvent?.contentIndex ?? 0;
+ const blocks = {
+ ...state.blocks,
+ [index]: (state.blocks[index] ?? "") + delta,
};
+ return replaceCurrentMessage({ ...state, blocks }, joinBlocks(blocks));
+ }
+
+ // Not every provider streams token-by-token. The screenpipe cloud route the
+ // summary Pipe actually runs on closes each block with `text_end` carrying
+ // the whole string and emits no `text_delta` at all, so a delta-only reader
+ // saw the summary as silence and the tab sat on its skeleton for the entire
+ // run. `text_end` is authoritative for its block: assigning rather than
+ // appending also keeps providers that send both from doubling the text.
+ if (
+ type === "message_update" &&
+ assistantEvent?.type === "text_end" &&
+ typeof assistantEvent.content === "string"
+ ) {
+ const index = assistantEvent.contentIndex ?? 0;
+ const blocks = { ...state.blocks, [index]: assistantEvent.content };
+ return replaceCurrentMessage({ ...state, blocks }, joinBlocks(blocks));
}
if (type === "message_end") {
const messageText = assistantMessageText(event.message);
- if (messageText) return replaceCurrentMessage(state, messageText);
+ if (messageText) {
+ return replaceCurrentMessage(
+ { ...state, blocks: { 0: messageText } },
+ messageText,
+ );
+ }
}
return state;
diff --git a/apps/screenpipe-app-tauri/lib/events/types.ts b/apps/screenpipe-app-tauri/lib/events/types.ts
index 86ea3f49b..8113dbcde 100644
--- a/apps/screenpipe-app-tauri/lib/events/types.ts
+++ b/apps/screenpipe-app-tauri/lib/events/types.ts
@@ -47,6 +47,9 @@ export interface AgentInnerEvent {
assistantMessageEvent?: {
type?: string;
delta?: string;
+ /** Full text of the block, carried on `text_end` by providers that do
+ * not emit incremental `text_delta` events. */
+ content?: string;
contentIndex?: number;
toolName?: string;
partial?: { content?: Array<{ type?: string; name?: string }> };
diff --git a/crates/screenpipe-core/assets/pipes/meeting-summary/pipe.md b/crates/screenpipe-core/assets/pipes/meeting-summary/pipe.md
index 0b91c3b38..a41a0a54a 100644
--- a/crates/screenpipe-core/assets/pipes/meeting-summary/pipe.md
+++ b/crates/screenpipe-core/assets/pipes/meeting-summary/pipe.md
@@ -64,9 +64,11 @@ step 2d — name the speakers from the screen (do this every run, don't ask firs
only rename when the on-screen evidence is unambiguous — never guess from voice alone. note which speakers you renamed (and which you left as-is) in your final message.
-step 3 — before saving, write the proposed summary in your response starting on a line with exactly `## Summary`. put only summary content after that heading and use that same markdown in `<YOUR_SUMMARY>`. the meeting UI streams this section while you write it, so do not put planning, tool narration, or save confirmations after the heading.
+step 3 — write the summary out as your own message, before you save it. this message must contain no tool call; end the turn after it. start a line with exactly `## Summary` and put the finished summary markdown after that heading. the meeting UI streams this section live while you write it — it is the only way the user sees anything before the run ends — and it is the same markdown you pass as `<YOUR_SUMMARY>` in step 3b.
-if your summary is worth saving, append it to the meeting note (and refresh the title in the same call) via:
+this step is not optional and it is not the closing report. "meeting 112 was summarized and saved to its record" is a report; it does not satisfy this step. saving a summary you never printed means the user watched a placeholder for the entire run and then got nothing to read, so treat that as a failed run. keep planning, tool narration, and save confirmations out of the section after the heading — those belong in your closing message in step 4.
+
+step 3b — now save it. if your summary is worth saving, append it to the meeting note (and refresh the title in the same call) via:
curl -s -X PUT "http://localhost:3030/meetings/<MEETING_ID>" \
-H "Authorization: Bearer $SCREENPIPE_LOCAL_API_KEY" \