YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(chat): send composer mentions as resolved context, not raw tokens (#5952)
Louis Beaumont2dd8b8810fe6
diff --git a/apps/screenpipe-app-tauri/components/__tests__/global-chat-mentions.test.ts b/apps/screenpipe-app-tauri/components/__tests__/global-chat-mentions.test.tsindex cde3afad8..d59039965 100644--- a/apps/screenpipe-app-tauri/components/__tests__/global-chat-mentions.test.ts+++ b/apps/screenpipe-app-tauri/components/__tests__/global-chat-mentions.test.ts@@ -12,7 +12,7 @@ import {findComposerMention,filterMentionSuggestions,mentionSuggestionIdentity,- normalizeComposerTimeRangesForModel,+ normalizeComposerMentionsForModel,parseMentions,resolvePinnedMentionIndex,TIME_RANGE_MENTION_SUGGESTIONS,@@ -306,7 +306,7 @@ describe("global chat mentions", () => {it("sends exact ISO boundaries instead of an ambiguous date token", () => {const expectedStart = new Date(2025, 3, 3, 0, 0, 0, 0).toISOString();const expectedEnd = new Date(2025, 3, 3, 23, 59, 59, 999).toISOString();- const normalized = normalizeComposerTimeRangesForModel(+ const normalized = normalizeComposerMentionsForModel("summarize activity ~(03/04/2025)",{ now: new Date(2026, 6, 30, 12, 0, 0) },);@@ -315,7 +315,25 @@ describe("global chat mentions", () => {expect(normalized.modelInput).toContain(`start_time: ${expectedStart}`);expect(normalized.modelInput).toContain(`end_time: ${expectedEnd}`);expect(normalized.modelInput).toContain("summarize activity");- expect(normalized.timeRanges).toHaveLength(1);+ expect(normalized.context.timeRanges).toHaveLength(1);+ });++ it("sends the filter chips as resolved values, not raw tokens", () => {+ // The regression: the chips said "audio / Slack / #project" while the+ // model only ever saw the literal characters and had to guess.+ const normalized = normalizeComposerMentionsForModel(+ "@audio @slack #project what did we decide",+ { now: new Date(2026, 6, 30, 12, 0, 0) },+ );++ expect(normalized.modelInput).toContain("content_type: audio");+ expect(normalized.modelInput).toContain("app_name: Slack");+ expect(normalized.modelInput).toContain("tags: project");++ const sentence = normalized.modelInput+ .split("</screenpipe_query_context>")[1]+ ?.trim();+ expect(sentence).toBe("what did we decide");});it("keeps a keyboard-selected filter pinned when recent chats arrive", () => {diff --git a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/pi-types.ts b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/pi-types.tsindex 94e8b5a5a..a1b87fcac 100644--- a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/pi-types.ts+++ b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/pi-types.ts@@ -11,6 +11,7 @@ import type {PiQueuedPrompt,} from "@/lib/utils/tauri";import type { ExtractedDoc } from "@/lib/pi/extract-document";+import type { ComposerMentionContext } from "@/lib/chat-utils";import type {ChatAttachment,ChatSendOptions,@@ -210,6 +211,15 @@ export type PiSendTransportOptions = {prefillContext: string | null;prefillFrameId: number | null;prefillSource: string;+ /**+ * Resolves composer mention tokens into a `<screenpipe_query_context>` block+ * before the turn is sent. Owned by the chat surface because it needs the+ * live app-tag map and the installed skill list. Omit it and the transport+ * falls back to the pure resolver (static app aliases only).+ */+ resolveComposerMentions?: (+ input: string,+ ) => Promise<{ modelInput: string; context: ComposerMentionContext }>;queuedPrompts: PiQueuedPrompt[];registerTurnIntent: TurnIntentActions["registerTurnIntent"];markTurnIntentConsumed: TurnIntentActions["markTurnIntentConsumed"];diff --git a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-chat-panel-effects.ts b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-chat-panel-effects.tsindex 3792969d9..d8843b3b1 100644--- a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-chat-panel-effects.ts+++ b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-chat-panel-effects.ts@@ -9,7 +9,14 @@ import { commands } from "@/lib/utils/tauri";interface UseChatPanelEffectsOptions {inputRef: React.RefObject<HTMLTextAreaElement>;- showMentionDropdown: boolean;+ /**+ * True only when the mention list is really on screen with at least one+ * suggestion. Must not be the raw "a trigger char was typed" flag: `~`+ * matches to end-of-line, so that flag stays true for the rest of the+ * message and would swallow Escape (no stop, no window close) long after+ * the list stopped rendering.+ */+ mentionDropdownIsOpen: boolean;isLoading: boolean;isStreaming: boolean;piActiveStopRequestedRef: React.MutableRefObject<boolean>;@@ -26,7 +33,7 @@ interface UseChatPanelEffectsOptions {export function useChatPanelEffects({inputRef,- showMentionDropdown,+ mentionDropdownIsOpen,isLoading,isStreaming,piActiveStopRequestedRef,@@ -45,7 +52,7 @@ export function useChatPanelEffects({}, [inputRef]);useEventListener("keydown", async (event: KeyboardEvent) => {- if (event.key !== "Escape" || showMentionDropdown) return;+ if (event.key !== "Escape" || mentionDropdownIsOpen) return;if (isLoading || isStreaming) {piActiveStopRequestedRef.current = true;try {diff --git a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-send-transport.ts b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-send-transport.tsindex 949eaadbd..50e9e0aee 100644--- a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-send-transport.ts+++ b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-send-transport.ts@@ -26,7 +26,7 @@ import {promptWithConversationHistory,} from "@/components/chat/standalone/hooks/pi-message-preparation";import type { ChatSendOptions, Message } from "@/lib/chat/types";-import { normalizeComposerTimeRangesForModel } from "@/lib/chat-utils";+import { normalizeComposerMentionsForModel } from "@/lib/chat-utils";import { chatSendTelemetryContext } from "@/lib/chat/response-feedback";import type { PiSendTransportOptions } from "@/components/chat/standalone/hooks/pi-types";@@ -108,6 +108,7 @@ export function usePiSendTransport(options: PiSendTransportOptions) {prefillContext,prefillFrameId,prefillSource,+ resolveComposerMentions,restartCurrentPiSession,saveConversation,sendDispatchInFlightRef,@@ -831,11 +832,21 @@ export function usePiSendTransport(options: PiSendTransportOptions) {) {if ((!canChat && !autoSendBypassRef.current) || (!getActivePreset() && !autoSendBypassRef.current)) return;const originalTrimmed = userMessage.trim();- const normalizedTimeRanges = normalizeComposerTimeRangesForModel(originalTrimmed);- const trimmed = normalizedTimeRanges.modelInput.trim();- const resolvedDisplayLabel = normalizedTimeRanges.timeRanges.length > 0- ? (displayLabel ?? originalTrimmed)- : displayLabel;+ // Composer mentions (`@app`, `@audio`, `@"speaker"`, `#tag`, `~range`,+ // `$skill`) are resolved into an explicit context block here. Without this+ // the model receives the raw token and has to guess what the chips above+ // the input already decided. `resolveComposerMentions` is supplied by the+ // chat surface because the app-tag map and installed skills live there;+ // the pure fallback still covers the static aliases.+ const normalizedMentions = resolveComposerMentions+ ? await resolveComposerMentions(originalTrimmed)+ : normalizeComposerMentionsForModel(originalTrimmed);+ const trimmed = normalizedMentions.modelInput.trim();+ // The bubble keeps the sentence the user actually typed; only the wire+ // payload carries the resolved block.+ const resolvedDisplayLabel = trimmed === originalTrimmed+ ? displayLabel+ : (displayLabel ?? originalTrimmed);const outgoingImages = imageDataUrls ?? pastedImages;const queuedDocs = attachedDocsRef.current;if (!trimmed && outgoingImages.length === 0 && queuedDocs.length === 0) return;diff --git a/apps/screenpipe-app-tauri/components/standalone-chat.tsx b/apps/screenpipe-app-tauri/components/standalone-chat.tsxindex ecf774ca8..79522a2b2 100644--- a/apps/screenpipe-app-tauri/components/standalone-chat.tsx+++ b/apps/screenpipe-app-tauri/components/standalone-chat.tsx@@ -39,8 +39,11 @@ import { loadConversationFile } from "@/lib/chat-storage";import {buildAppMentionSuggestions,buildTagMentionSuggestions,+ buildComposerSkillReferences,isConversationHistorySyncPrompt,isInjectedTitleSourcePrompt,+ normalizeComposerMentionsForModel,+ type ComposerSkillReference,} from "@/lib/chat-utils";import { useAutoSuggestions } from "@/lib/hooks/use-auto-suggestions";import {@@ -364,6 +367,29 @@ export function StandaloneChat({}return map;}, [appMentionSuggestions]);+ // Installed skills are only needed to turn a typed `$name` into a loadable+ // path, so they are fetched on the first turn that actually contains a `$`+ // token and cached for the rest of the session.+ const composerSkillsRef = useRef<ComposerSkillReference[] | null>(null);+ const resolveComposerMentions = React.useCallback(+ async (text: string) => {+ let skills = composerSkillsRef.current ?? [];+ if (/(^|\s)\$[\w:.-]+/.test(text) && composerSkillsRef.current === null) {+ try {+ const result = await commands.listImportedSkills();+ skills = result.status === "ok"+ ? buildComposerSkillReferences(result.data)+ : [];+ } catch {+ skills = [];+ }+ composerSkillsRef.current = skills;+ }+ return normalizeComposerMentionsForModel(text, { appTagMap, skills });+ },+ [appTagMap],+ );+const openMentionConversationRef = useRef<((conversationId: string) => void | Promise<void>) | null>(null);@@ -1016,7 +1042,7 @@ export function StandaloneChat({useChatPanelEffects({inputRef,- showMentionDropdown,+ mentionDropdownIsOpen: showMentionDropdown && filteredMentions.length > 0,isLoading,isStreaming,piActiveStopRequestedRef,@@ -1091,6 +1117,7 @@ export function StandaloneChat({prefillContext,prefillFrameId,prefillSource,+ resolveComposerMentions,queuedPrompts,registerTurnIntent,markTurnIntentConsumed,diff --git a/apps/screenpipe-app-tauri/lib/chat-utils.test.ts b/apps/screenpipe-app-tauri/lib/chat-utils.test.tsindex 60e148893..f323e7806 100644--- a/apps/screenpipe-app-tauri/lib/chat-utils.test.ts+++ b/apps/screenpipe-app-tauri/lib/chat-utils.test.ts@@ -37,6 +37,7 @@ import {shouldActivateHomeSectionForChatLoadConversation,shouldHandleChatLoadConversationForWindow,shouldHandleChatPrefillForWindow,+ normalizeComposerMentionsForModel,} from "./chat-utils";import { useChatStore } from "./stores/chat-store";@@ -177,3 +178,71 @@ describe("shouldHandleChatPrefillForWindow", () => {expect(shouldHandleChatPrefillForWindow(undefined, "chat")).toBe(false);});});++describe("normalizeComposerMentionsForModel", () => {+ const now = new Date("2026-08-06T12:00:00.000Z");+ const skills = [+ { name: "deep-research", path: "/Users/u/.screenpipe/skills/deep-research" },+ ];++ it("leaves a plain message untouched", () => {+ const result = normalizeComposerMentionsForModel("what did i work on", { now });+ expect(result.modelInput).toBe("what did i work on");+ expect(result.context.contentType).toBeNull();+ expect(result.context.skills).toEqual([]);+ });++ it("resolves content type, app and tags into one context block", () => {+ const result = normalizeComposerMentionsForModel(+ "@audio @slack #project what did we decide",+ { now },+ );+ expect(result.modelInput).toContain("<screenpipe_query_context>");+ expect(result.modelInput).toContain("content_type: audio");+ expect(result.modelInput).toContain("app_name: Slack");+ expect(result.modelInput).toContain("tags: project");+ expect(result.context.contentType).toBe("audio");+ expect(result.context.appName).toBe("Slack");+ expect(result.context.tagNames).toEqual(["project"]);+ });++ it("strips the resolved tokens from the sentence the model reads", () => {+ const result = normalizeComposerMentionsForModel("@audio #project summarize", { now });+ const sentence = result.modelInput.split("</screenpipe_query_context>")[1]?.trim();+ expect(sentence).toBe("summarize");+ expect(sentence).not.toContain("@audio");+ expect(sentence).not.toContain("#project");+ });++ it("emits exact ISO boundaries for a time range", () => {+ const result = normalizeComposerMentionsForModel("~lastweek recap", { now });+ expect(result.context.timeRanges).toHaveLength(1);+ expect(result.modelInput).toContain("start_time: ");+ expect(result.modelInput).toContain("end_time: ");+ expect(result.modelInput).toContain(result.context.timeRanges[0].startTime);+ });++ it("resolves a quoted speaker mention", () => {+ const result = normalizeComposerMentionsForModel('@"John Doe" what did he say', { now });+ expect(result.context.speakerName).toBe("John Doe");+ expect(result.modelInput).toContain("speaker: John Doe");+ });++ it("turns a known skill token into a loadable path", () => {+ const result = normalizeComposerMentionsForModel("$deep-research on this", {+ now,+ skills,+ });+ expect(result.context.skills).toEqual(skills);+ expect(result.modelInput).toContain("path: /Users/u/.screenpipe/skills/deep-research");+ expect(result.modelInput).toContain("Load each listed skill from its path before answering.");+ const sentence = result.modelInput.split("</screenpipe_query_context>")[1]?.trim();+ expect(sentence).toBe("on this");+ });++ it("keeps an unknown skill token in the sentence instead of dropping it", () => {+ const result = normalizeComposerMentionsForModel("$nope on this", { now, skills });+ expect(result.context.skills).toEqual([]);+ expect(result.modelInput).toBe("$nope on this");+ });+});diff --git a/apps/screenpipe-app-tauri/lib/chat-utils.ts b/apps/screenpipe-app-tauri/lib/chat-utils.tsindex 8b969855e..e0328f08b 100644--- a/apps/screenpipe-app-tauri/lib/chat-utils.ts+++ b/apps/screenpipe-app-tauri/lib/chat-utils.ts@@ -577,43 +577,143 @@ export interface ComposerTimeRangeContext {endTime: string;}-export function normalizeComposerTimeRangesForModel(+export interface ComposerSkillReference {+ /** Folder key used as the `$tag`, e.g. `deep-research` for `$deep-research`. */+ name: string;+ /** Absolute path to the skill so the agent can load it without searching. */+ path: string;+}++export interface ComposerMentionContext {+ timeRanges: ComposerTimeRangeContext[];+ contentType: string | null;+ appName: string | null;+ speakerName: string | null;+ tagNames: string[];+ skills: ComposerSkillReference[];+}++export interface NormalizeComposerMentionsOptions extends ParseMentionsOptions {+ /** Installed skills, used to turn a bare `$name` into a loadable path. */+ skills?: ComposerSkillReference[];+}++const SKILL_MENTION_PATTERN = /(^|\s)\$([\w:.-]+)/g;++function emptyMentionContext(): ComposerMentionContext {+ return {+ timeRanges: [],+ contentType: null,+ appName: null,+ speakerName: null,+ tagNames: [],+ skills: [],+ };+}++/**+ * Turn composer mention tokens into an explicit contract the model can act on.+ *+ * The composer is plain text, so `@audio`, `@slack`, `@"John Doe"`, `#tag`,+ * `~lastweek` and `$skill` are just characters by the time a turn is sent. The+ * chips above the input promise a filter; without this step the model only+ * receives the raw token and has to guess what it meant. That guess is the bug:+ * a mention that renders as an active filter must arrive as a resolved value.+ *+ * So every token the UI already resolved for its chips is resolved again here,+ * emitted as a `<screenpipe_query_context>` block, and removed from the+ * user-visible sentence (the same split Claude Code and Codex use: resolved+ * context travels beside the prompt, never inside it). Tokens that could not be+ * resolved are left untouched so nothing silently disappears.+ */+export function normalizeComposerMentionsForModel(input: string,- options?: { now?: Date },+ options?: NormalizeComposerMentionsOptions,): {modelInput: string;- timeRanges: ComposerTimeRangeContext[];+ context: ComposerMentionContext;} {- const now = options?.now ? new Date(options.now) : new Date();- const parsed = parseExplicitTimeRanges(input, now);- const timeRanges = parsed.ranges.map((range) => ({+ const parsed = parseMentions(input, options);+ const timeRanges = parsed.timeRanges.map((range) => ({label: range.label,startTime: range.start.toISOString(),endTime: range.end.toISOString(),}));- if (timeRanges.length === 0) {- return { modelInput: input, timeRanges };+ // `$skill` is an action, not a filter, so parseMentions leaves it alone.+ // Resolve it against the installed set; an unknown `$foo` stays in the+ // sentence rather than vanishing into a context block that means nothing.+ const installedSkills = options?.skills ?? [];+ const skills: ComposerSkillReference[] = [];+ let cleanedInput = parsed.cleanedInput;+ if (installedSkills.length > 0) {+ cleanedInput = cleanedInput.replace(+ SKILL_MENTION_PATTERN,+ (match, leading: string, name: string) => {+ const skill = installedSkills.find(+ (candidate) => candidate.name.toLowerCase() === name.toLowerCase(),+ );+ if (!skill) return match;+ if (!skills.some((existing) => existing.name === skill.name)) {+ skills.push(skill);+ }+ return leading;+ },+ );}- const queryContext = timeRanges.flatMap((range, index) => [- `time_range_${index + 1}:`,- ` label: ${range.label}`,- ` start_time: ${range.startTime}`,- ` end_time: ${range.endTime}`,- ]);+ const context: ComposerMentionContext = {+ timeRanges,+ contentType: parsed.contentType,+ appName: parsed.appName,+ speakerName: parsed.speakerName,+ tagNames: parsed.tagNames,+ skills,+ };++ const hasResolvedMention =+ timeRanges.length > 0 ||+ Boolean(parsed.contentType) ||+ Boolean(parsed.appName) ||+ Boolean(parsed.speakerName) ||+ parsed.tagNames.length > 0 ||+ skills.length > 0;++ if (!hasResolvedMention) {+ return { modelInput: input, context: emptyMentionContext() };+ }++ const lines: string[] = [];+ for (const [index, range] of timeRanges.entries()) {+ lines.push(+ `time_range_${index + 1}:`,+ ` label: ${range.label}`,+ ` start_time: ${range.startTime}`,+ ` end_time: ${range.endTime}`,+ );+ }+ if (context.contentType) lines.push(`content_type: ${context.contentType}`);+ if (context.appName) lines.push(`app_name: ${context.appName}`);+ if (context.speakerName) lines.push(`speaker: ${context.speakerName}`);+ if (context.tagNames.length > 0) lines.push(`tags: ${context.tagNames.join(", ")}`);+ for (const [index, skill] of skills.entries()) {+ lines.push(`skill_${index + 1}:`, ` name: ${skill.name}`, ` path: ${skill.path}`);+ }+ if (skills.length > 0) {+ lines.push("Load each listed skill from its path before answering.");+ }return {modelInput: ["<screenpipe_query_context>",- "Use these exact ISO 8601 boundaries for screenpipe queries; do not reinterpret the original date token.",- ...queryContext,+ "The user selected these filters in the composer. Use these exact values when querying screenpipe; do not reinterpret the original tokens.",+ ...lines,"</screenpipe_query_context>",- parsed.cleanedInput,+ cleanedInput.replace(/\s+/g, " ").trim(),].filter(Boolean).join("\n"),- timeRanges,+ context,};}@@ -890,18 +990,32 @@ export function buildChatMentionSuggestions(}));}+/**+ * The `$tag` a skill is addressed by. Shared so the dropdown and the send-time+ * resolver agree on the key; if they drift, `$name` stops resolving to a path.+ */+export function skillMentionKey(item: SkillMentionItem): string {+ const pathParts = item.path.split(/[\\/]/).filter(Boolean);+ const folderName = pathParts.at(-1);+ const fallbackName = item.name+ .trim()+ .toLowerCase()+ .replace(/[^a-z0-9._-]+/g, "-")+ .replace(/^-+|-+$/g, "");+ return folderName || fallbackName || "skill";+}++export function buildComposerSkillReferences(+ items: SkillMentionItem[],+): ComposerSkillReference[] {+ return items.map((item) => ({ name: skillMentionKey(item), path: item.path }));+}+export function buildSkillMentionSuggestions(items: SkillMentionItem[],): MentionSuggestion[] {return items.map((item) => {- const pathParts = item.path.split(/[\\/]/).filter(Boolean);- const folderName = pathParts.at(-1);- const fallbackName = item.name- .trim()- .toLowerCase()- .replace(/[^a-z0-9._-]+/g, "-")- .replace(/^-+|-+$/g, "");- const skillKey = folderName || fallbackName || "skill";+ const skillKey = skillMentionKey(item);return {tag: `$${skillKey}`,label: item.name.trim() || skillKey,