Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(chat): send composer mentions as resolved context, not raw tokens (#5952)

Louis Beaumont2dd8b8810fe6
diff --git a/apps/screenpipe-app-tauri/components/__tests__/global-chat-mentions.test.ts b/apps/screenpipe-app-tauri/components/__tests__/global-chat-mentions.test.ts
index cde3afad8..d59039965 100644
--- a/apps/screenpipe-app-tauri/components/__tests__/global-chat-mentions.test.ts
+++ b/apps/screenpipe-app-tauri/components/__tests__/global-chat-mentions.test.ts
@@ -12,7 +12,7 @@ import {
findComposerMention,
filterMentionSuggestions,
mentionSuggestionIdentity,
- normalizeComposerTimeRangesForModel,
+ normalizeComposerMentionsForModel,
parseMentions,
resolvePinnedMentionIndex,
TIME_RANGE_MENTION_SUGGESTIONS,
@@ -306,7 +306,7 @@ describe("global chat mentions", () => {
it("sends exact ISO boundaries instead of an ambiguous date token", () => {
const expectedStart = new Date(2025, 3, 3, 0, 0, 0, 0).toISOString();
const expectedEnd = new Date(2025, 3, 3, 23, 59, 59, 999).toISOString();
- const normalized = normalizeComposerTimeRangesForModel(
+ const normalized = normalizeComposerMentionsForModel(
"summarize activity ~(03/04/2025)",
{ now: new Date(2026, 6, 30, 12, 0, 0) },
);
@@ -315,7 +315,25 @@ describe("global chat mentions", () => {
expect(normalized.modelInput).toContain(`start_time: ${expectedStart}`);
expect(normalized.modelInput).toContain(`end_time: ${expectedEnd}`);
expect(normalized.modelInput).toContain("summarize activity");
- expect(normalized.timeRanges).toHaveLength(1);
+ expect(normalized.context.timeRanges).toHaveLength(1);
+ });
+
+ it("sends the filter chips as resolved values, not raw tokens", () => {
+ // The regression: the chips said "audio / Slack / #project" while the
+ // model only ever saw the literal characters and had to guess.
+ const normalized = normalizeComposerMentionsForModel(
+ "@audio @slack #project what did we decide",
+ { now: new Date(2026, 6, 30, 12, 0, 0) },
+ );
+
+ expect(normalized.modelInput).toContain("content_type: audio");
+ expect(normalized.modelInput).toContain("app_name: Slack");
+ expect(normalized.modelInput).toContain("tags: project");
+
+ const sentence = normalized.modelInput
+ .split("</screenpipe_query_context>")[1]
+ ?.trim();
+ expect(sentence).toBe("what did we decide");
});
it("keeps a keyboard-selected filter pinned when recent chats arrive", () => {
diff --git a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/pi-types.ts b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/pi-types.ts
index 94e8b5a5a..a1b87fcac 100644
--- a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/pi-types.ts
+++ b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/pi-types.ts
@@ -11,6 +11,7 @@ import type {
PiQueuedPrompt,
} from "@/lib/utils/tauri";
import type { ExtractedDoc } from "@/lib/pi/extract-document";
+import type { ComposerMentionContext } from "@/lib/chat-utils";
import type {
ChatAttachment,
ChatSendOptions,
@@ -210,6 +211,15 @@ export type PiSendTransportOptions = {
prefillContext: string | null;
prefillFrameId: number | null;
prefillSource: string;
+ /**
+ * Resolves composer mention tokens into a `<screenpipe_query_context>` block
+ * before the turn is sent. Owned by the chat surface because it needs the
+ * live app-tag map and the installed skill list. Omit it and the transport
+ * falls back to the pure resolver (static app aliases only).
+ */
+ resolveComposerMentions?: (
+ input: string,
+ ) => Promise<{ modelInput: string; context: ComposerMentionContext }>;
queuedPrompts: PiQueuedPrompt[];
registerTurnIntent: TurnIntentActions["registerTurnIntent"];
markTurnIntentConsumed: TurnIntentActions["markTurnIntentConsumed"];
diff --git a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-chat-panel-effects.ts b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-chat-panel-effects.ts
index 3792969d9..d8843b3b1 100644
--- a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-chat-panel-effects.ts
+++ b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-chat-panel-effects.ts
@@ -9,7 +9,14 @@ import { commands } from "@/lib/utils/tauri";
interface UseChatPanelEffectsOptions {
inputRef: React.RefObject<HTMLTextAreaElement>;
- showMentionDropdown: boolean;
+ /**
+ * True only when the mention list is really on screen with at least one
+ * suggestion. Must not be the raw "a trigger char was typed" flag: `~`
+ * matches to end-of-line, so that flag stays true for the rest of the
+ * message and would swallow Escape (no stop, no window close) long after
+ * the list stopped rendering.
+ */
+ mentionDropdownIsOpen: boolean;
isLoading: boolean;
isStreaming: boolean;
piActiveStopRequestedRef: React.MutableRefObject<boolean>;
@@ -26,7 +33,7 @@ interface UseChatPanelEffectsOptions {
export function useChatPanelEffects({
inputRef,
- showMentionDropdown,
+ mentionDropdownIsOpen,
isLoading,
isStreaming,
piActiveStopRequestedRef,
@@ -45,7 +52,7 @@ export function useChatPanelEffects({
}, [inputRef]);
useEventListener("keydown", async (event: KeyboardEvent) => {
- if (event.key !== "Escape" || showMentionDropdown) return;
+ if (event.key !== "Escape" || mentionDropdownIsOpen) return;
if (isLoading || isStreaming) {
piActiveStopRequestedRef.current = true;
try {
diff --git a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-send-transport.ts b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-send-transport.ts
index 949eaadbd..50e9e0aee 100644
--- a/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-send-transport.ts
+++ b/apps/screenpipe-app-tauri/components/chat/standalone/hooks/use-pi-send-transport.ts
@@ -26,7 +26,7 @@ import {
promptWithConversationHistory,
} from "@/components/chat/standalone/hooks/pi-message-preparation";
import type { ChatSendOptions, Message } from "@/lib/chat/types";
-import { normalizeComposerTimeRangesForModel } from "@/lib/chat-utils";
+import { normalizeComposerMentionsForModel } from "@/lib/chat-utils";
import { chatSendTelemetryContext } from "@/lib/chat/response-feedback";
import type { PiSendTransportOptions } from "@/components/chat/standalone/hooks/pi-types";
@@ -108,6 +108,7 @@ export function usePiSendTransport(options: PiSendTransportOptions) {
prefillContext,
prefillFrameId,
prefillSource,
+ resolveComposerMentions,
restartCurrentPiSession,
saveConversation,
sendDispatchInFlightRef,
@@ -831,11 +832,21 @@ export function usePiSendTransport(options: PiSendTransportOptions) {
) {
if ((!canChat && !autoSendBypassRef.current) || (!getActivePreset() && !autoSendBypassRef.current)) return;
const originalTrimmed = userMessage.trim();
- const normalizedTimeRanges = normalizeComposerTimeRangesForModel(originalTrimmed);
- const trimmed = normalizedTimeRanges.modelInput.trim();
- const resolvedDisplayLabel = normalizedTimeRanges.timeRanges.length > 0
- ? (displayLabel ?? originalTrimmed)
- : displayLabel;
+ // Composer mentions (`@app`, `@audio`, `@"speaker"`, `#tag`, `~range`,
+ // `$skill`) are resolved into an explicit context block here. Without this
+ // the model receives the raw token and has to guess what the chips above
+ // the input already decided. `resolveComposerMentions` is supplied by the
+ // chat surface because the app-tag map and installed skills live there;
+ // the pure fallback still covers the static aliases.
+ const normalizedMentions = resolveComposerMentions
+ ? await resolveComposerMentions(originalTrimmed)
+ : normalizeComposerMentionsForModel(originalTrimmed);
+ const trimmed = normalizedMentions.modelInput.trim();
+ // The bubble keeps the sentence the user actually typed; only the wire
+ // payload carries the resolved block.
+ const resolvedDisplayLabel = trimmed === originalTrimmed
+ ? displayLabel
+ : (displayLabel ?? originalTrimmed);
const outgoingImages = imageDataUrls ?? pastedImages;
const queuedDocs = attachedDocsRef.current;
if (!trimmed && outgoingImages.length === 0 && queuedDocs.length === 0) return;
diff --git a/apps/screenpipe-app-tauri/components/standalone-chat.tsx b/apps/screenpipe-app-tauri/components/standalone-chat.tsx
index ecf774ca8..79522a2b2 100644
--- a/apps/screenpipe-app-tauri/components/standalone-chat.tsx
+++ b/apps/screenpipe-app-tauri/components/standalone-chat.tsx
@@ -39,8 +39,11 @@ import { loadConversationFile } from "@/lib/chat-storage";
import {
buildAppMentionSuggestions,
buildTagMentionSuggestions,
+ buildComposerSkillReferences,
isConversationHistorySyncPrompt,
isInjectedTitleSourcePrompt,
+ normalizeComposerMentionsForModel,
+ type ComposerSkillReference,
} from "@/lib/chat-utils";
import { useAutoSuggestions } from "@/lib/hooks/use-auto-suggestions";
import {
@@ -364,6 +367,29 @@ export function StandaloneChat({
}
return map;
}, [appMentionSuggestions]);
+ // Installed skills are only needed to turn a typed `$name` into a loadable
+ // path, so they are fetched on the first turn that actually contains a `$`
+ // token and cached for the rest of the session.
+ const composerSkillsRef = useRef<ComposerSkillReference[] | null>(null);
+ const resolveComposerMentions = React.useCallback(
+ async (text: string) => {
+ let skills = composerSkillsRef.current ?? [];
+ if (/(^|\s)\$[\w:.-]+/.test(text) && composerSkillsRef.current === null) {
+ try {
+ const result = await commands.listImportedSkills();
+ skills = result.status === "ok"
+ ? buildComposerSkillReferences(result.data)
+ : [];
+ } catch {
+ skills = [];
+ }
+ composerSkillsRef.current = skills;
+ }
+ return normalizeComposerMentionsForModel(text, { appTagMap, skills });
+ },
+ [appTagMap],
+ );
+
const openMentionConversationRef = useRef<
((conversationId: string) => void | Promise<void>) | null
>(null);
@@ -1016,7 +1042,7 @@ export function StandaloneChat({
useChatPanelEffects({
inputRef,
- showMentionDropdown,
+ mentionDropdownIsOpen: showMentionDropdown && filteredMentions.length > 0,
isLoading,
isStreaming,
piActiveStopRequestedRef,
@@ -1091,6 +1117,7 @@ export function StandaloneChat({
prefillContext,
prefillFrameId,
prefillSource,
+ resolveComposerMentions,
queuedPrompts,
registerTurnIntent,
markTurnIntentConsumed,
diff --git a/apps/screenpipe-app-tauri/lib/chat-utils.test.ts b/apps/screenpipe-app-tauri/lib/chat-utils.test.ts
index 60e148893..f323e7806 100644
--- a/apps/screenpipe-app-tauri/lib/chat-utils.test.ts
+++ b/apps/screenpipe-app-tauri/lib/chat-utils.test.ts
@@ -37,6 +37,7 @@ import {
shouldActivateHomeSectionForChatLoadConversation,
shouldHandleChatLoadConversationForWindow,
shouldHandleChatPrefillForWindow,
+ normalizeComposerMentionsForModel,
} from "./chat-utils";
import { useChatStore } from "./stores/chat-store";
@@ -177,3 +178,71 @@ describe("shouldHandleChatPrefillForWindow", () => {
expect(shouldHandleChatPrefillForWindow(undefined, "chat")).toBe(false);
});
});
+
+describe("normalizeComposerMentionsForModel", () => {
+ const now = new Date("2026-08-06T12:00:00.000Z");
+ const skills = [
+ { name: "deep-research", path: "/Users/u/.screenpipe/skills/deep-research" },
+ ];
+
+ it("leaves a plain message untouched", () => {
+ const result = normalizeComposerMentionsForModel("what did i work on", { now });
+ expect(result.modelInput).toBe("what did i work on");
+ expect(result.context.contentType).toBeNull();
+ expect(result.context.skills).toEqual([]);
+ });
+
+ it("resolves content type, app and tags into one context block", () => {
+ const result = normalizeComposerMentionsForModel(
+ "@audio @slack #project what did we decide",
+ { now },
+ );
+ expect(result.modelInput).toContain("<screenpipe_query_context>");
+ expect(result.modelInput).toContain("content_type: audio");
+ expect(result.modelInput).toContain("app_name: Slack");
+ expect(result.modelInput).toContain("tags: project");
+ expect(result.context.contentType).toBe("audio");
+ expect(result.context.appName).toBe("Slack");
+ expect(result.context.tagNames).toEqual(["project"]);
+ });
+
+ it("strips the resolved tokens from the sentence the model reads", () => {
+ const result = normalizeComposerMentionsForModel("@audio #project summarize", { now });
+ const sentence = result.modelInput.split("</screenpipe_query_context>")[1]?.trim();
+ expect(sentence).toBe("summarize");
+ expect(sentence).not.toContain("@audio");
+ expect(sentence).not.toContain("#project");
+ });
+
+ it("emits exact ISO boundaries for a time range", () => {
+ const result = normalizeComposerMentionsForModel("~lastweek recap", { now });
+ expect(result.context.timeRanges).toHaveLength(1);
+ expect(result.modelInput).toContain("start_time: ");
+ expect(result.modelInput).toContain("end_time: ");
+ expect(result.modelInput).toContain(result.context.timeRanges[0].startTime);
+ });
+
+ it("resolves a quoted speaker mention", () => {
+ const result = normalizeComposerMentionsForModel('@"John Doe" what did he say', { now });
+ expect(result.context.speakerName).toBe("John Doe");
+ expect(result.modelInput).toContain("speaker: John Doe");
+ });
+
+ it("turns a known skill token into a loadable path", () => {
+ const result = normalizeComposerMentionsForModel("$deep-research on this", {
+ now,
+ skills,
+ });
+ expect(result.context.skills).toEqual(skills);
+ expect(result.modelInput).toContain("path: /Users/u/.screenpipe/skills/deep-research");
+ expect(result.modelInput).toContain("Load each listed skill from its path before answering.");
+ const sentence = result.modelInput.split("</screenpipe_query_context>")[1]?.trim();
+ expect(sentence).toBe("on this");
+ });
+
+ it("keeps an unknown skill token in the sentence instead of dropping it", () => {
+ const result = normalizeComposerMentionsForModel("$nope on this", { now, skills });
+ expect(result.context.skills).toEqual([]);
+ expect(result.modelInput).toBe("$nope on this");
+ });
+});
diff --git a/apps/screenpipe-app-tauri/lib/chat-utils.ts b/apps/screenpipe-app-tauri/lib/chat-utils.ts
index 8b969855e..e0328f08b 100644
--- a/apps/screenpipe-app-tauri/lib/chat-utils.ts
+++ b/apps/screenpipe-app-tauri/lib/chat-utils.ts
@@ -577,43 +577,143 @@ export interface ComposerTimeRangeContext {
endTime: string;
}
-export function normalizeComposerTimeRangesForModel(
+export interface ComposerSkillReference {
+ /** Folder key used as the `$tag`, e.g. `deep-research` for `$deep-research`. */
+ name: string;
+ /** Absolute path to the skill so the agent can load it without searching. */
+ path: string;
+}
+
+export interface ComposerMentionContext {
+ timeRanges: ComposerTimeRangeContext[];
+ contentType: string | null;
+ appName: string | null;
+ speakerName: string | null;
+ tagNames: string[];
+ skills: ComposerSkillReference[];
+}
+
+export interface NormalizeComposerMentionsOptions extends ParseMentionsOptions {
+ /** Installed skills, used to turn a bare `$name` into a loadable path. */
+ skills?: ComposerSkillReference[];
+}
+
+const SKILL_MENTION_PATTERN = /(^|\s)\$([\w:.-]+)/g;
+
+function emptyMentionContext(): ComposerMentionContext {
+ return {
+ timeRanges: [],
+ contentType: null,
+ appName: null,
+ speakerName: null,
+ tagNames: [],
+ skills: [],
+ };
+}
+
+/**
+ * Turn composer mention tokens into an explicit contract the model can act on.
+ *
+ * The composer is plain text, so `@audio`, `@slack`, `@"John Doe"`, `#tag`,
+ * `~lastweek` and `$skill` are just characters by the time a turn is sent. The
+ * chips above the input promise a filter; without this step the model only
+ * receives the raw token and has to guess what it meant. That guess is the bug:
+ * a mention that renders as an active filter must arrive as a resolved value.
+ *
+ * So every token the UI already resolved for its chips is resolved again here,
+ * emitted as a `<screenpipe_query_context>` block, and removed from the
+ * user-visible sentence (the same split Claude Code and Codex use: resolved
+ * context travels beside the prompt, never inside it). Tokens that could not be
+ * resolved are left untouched so nothing silently disappears.
+ */
+export function normalizeComposerMentionsForModel(
input: string,
- options?: { now?: Date },
+ options?: NormalizeComposerMentionsOptions,
): {
modelInput: string;
- timeRanges: ComposerTimeRangeContext[];
+ context: ComposerMentionContext;
} {
- const now = options?.now ? new Date(options.now) : new Date();
- const parsed = parseExplicitTimeRanges(input, now);
- const timeRanges = parsed.ranges.map((range) => ({
+ const parsed = parseMentions(input, options);
+ const timeRanges = parsed.timeRanges.map((range) => ({
label: range.label,
startTime: range.start.toISOString(),
endTime: range.end.toISOString(),
}));
- if (timeRanges.length === 0) {
- return { modelInput: input, timeRanges };
+ // `$skill` is an action, not a filter, so parseMentions leaves it alone.
+ // Resolve it against the installed set; an unknown `$foo` stays in the
+ // sentence rather than vanishing into a context block that means nothing.
+ const installedSkills = options?.skills ?? [];
+ const skills: ComposerSkillReference[] = [];
+ let cleanedInput = parsed.cleanedInput;
+ if (installedSkills.length > 0) {
+ cleanedInput = cleanedInput.replace(
+ SKILL_MENTION_PATTERN,
+ (match, leading: string, name: string) => {
+ const skill = installedSkills.find(
+ (candidate) => candidate.name.toLowerCase() === name.toLowerCase(),
+ );
+ if (!skill) return match;
+ if (!skills.some((existing) => existing.name === skill.name)) {
+ skills.push(skill);
+ }
+ return leading;
+ },
+ );
}
- const queryContext = timeRanges.flatMap((range, index) => [
- `time_range_${index + 1}:`,
- ` label: ${range.label}`,
- ` start_time: ${range.startTime}`,
- ` end_time: ${range.endTime}`,
- ]);
+ const context: ComposerMentionContext = {
+ timeRanges,
+ contentType: parsed.contentType,
+ appName: parsed.appName,
+ speakerName: parsed.speakerName,
+ tagNames: parsed.tagNames,
+ skills,
+ };
+
+ const hasResolvedMention =
+ timeRanges.length > 0 ||
+ Boolean(parsed.contentType) ||
+ Boolean(parsed.appName) ||
+ Boolean(parsed.speakerName) ||
+ parsed.tagNames.length > 0 ||
+ skills.length > 0;
+
+ if (!hasResolvedMention) {
+ return { modelInput: input, context: emptyMentionContext() };
+ }
+
+ const lines: string[] = [];
+ for (const [index, range] of timeRanges.entries()) {
+ lines.push(
+ `time_range_${index + 1}:`,
+ ` label: ${range.label}`,
+ ` start_time: ${range.startTime}`,
+ ` end_time: ${range.endTime}`,
+ );
+ }
+ if (context.contentType) lines.push(`content_type: ${context.contentType}`);
+ if (context.appName) lines.push(`app_name: ${context.appName}`);
+ if (context.speakerName) lines.push(`speaker: ${context.speakerName}`);
+ if (context.tagNames.length > 0) lines.push(`tags: ${context.tagNames.join(", ")}`);
+ for (const [index, skill] of skills.entries()) {
+ lines.push(`skill_${index + 1}:`, ` name: ${skill.name}`, ` path: ${skill.path}`);
+ }
+ if (skills.length > 0) {
+ lines.push("Load each listed skill from its path before answering.");
+ }
return {
modelInput: [
"<screenpipe_query_context>",
- "Use these exact ISO 8601 boundaries for screenpipe queries; do not reinterpret the original date token.",
- ...queryContext,
+ "The user selected these filters in the composer. Use these exact values when querying screenpipe; do not reinterpret the original tokens.",
+ ...lines,
"</screenpipe_query_context>",
- parsed.cleanedInput,
+ cleanedInput.replace(/\s+/g, " ").trim(),
]
.filter(Boolean)
.join("\n"),
- timeRanges,
+ context,
};
}
@@ -890,18 +990,32 @@ export function buildChatMentionSuggestions(
}));
}
+/**
+ * The `$tag` a skill is addressed by. Shared so the dropdown and the send-time
+ * resolver agree on the key; if they drift, `$name` stops resolving to a path.
+ */
+export function skillMentionKey(item: SkillMentionItem): string {
+ const pathParts = item.path.split(/[\\/]/).filter(Boolean);
+ const folderName = pathParts.at(-1);
+ const fallbackName = item.name
+ .trim()
+ .toLowerCase()
+ .replace(/[^a-z0-9._-]+/g, "-")
+ .replace(/^-+|-+$/g, "");
+ return folderName || fallbackName || "skill";
+}
+
+export function buildComposerSkillReferences(
+ items: SkillMentionItem[],
+): ComposerSkillReference[] {
+ return items.map((item) => ({ name: skillMentionKey(item), path: item.path }));
+}
+
export function buildSkillMentionSuggestions(
items: SkillMentionItem[],
): MentionSuggestion[] {
return items.map((item) => {
- const pathParts = item.path.split(/[\\/]/).filter(Boolean);
- const folderName = pathParts.at(-1);
- const fallbackName = item.name
- .trim()
- .toLowerCase()
- .replace(/[^a-z0-9._-]+/g, "-")
- .replace(/^-+|-+$/g, "");
- const skillKey = folderName || fallbackName || "skill";
+ const skillKey = skillMentionKey(item);
return {
tag: `$${skillKey}`,
label: item.name.trim() || skillKey,