YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(app): polish structured context setting (#5534)
Louis Beaumont55daa12869e2
diff --git a/.github/pr-media/structured-app-context-setting.png b/.github/pr-media/structured-app-context-setting.pngnew file mode 100644index 000000000..e98d3b28bBinary files /dev/null and b/.github/pr-media/structured-app-context-setting.png differdiff --git a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsxindex c65d6cd5e..98fbd821b 100644--- a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx+++ b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx@@ -40,8 +40,8 @@ export const searchIndex: SettingsField[] = [{ label: "Smart recording", keywords: ["smart recording", "beta", "meeting", "piggyback", "per-process", "meeting audio"], conditional: true },{ label: "Bluetooth microphones", keywords: ["bluetooth", "airpods", "headset", "a2dp", "sco", "meeting"], conditional: true },{ label: "Screen context capture", keywords: ["screen", "video", "accessibility"] },- { label: "structured app context", keywords: ["semantic", "ai", "messages", "email", "tasks", "code"], conditional: true },- { label: "AI context use", keywords: ["memory", "computer use", "automation", "agent", "skills"], conditional: true },+ { label: "Structured app context", keywords: ["semantic", "ai", "messages", "email", "tasks", "code"], conditional: true },+ { label: "Use it for", keywords: ["memory", "computer use", "automation", "agent", "skills"], conditional: true },{ label: "Screenshot images", keywords: ["screenshot", "pixels", "ocr", "jpeg"] },{ label: "Use all monitors", keywords: ["monitor", "display"], conditional: true },// conditional: monitor picker only renders when "Use all monitors" is off — paired right under that toggle.@@ -3788,13 +3788,13 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre<AppWindowMac className="h-4 w-4 text-muted-foreground shrink-0" /><div className="min-w-0"><h3 className="text-sm font-medium text-foreground flex items-center gap-1.5">- structured app context- <Badge variant="secondary" aria-label="experimental" className="px-1.5 py-0 text-[10px] font-medium uppercase tracking-wide">- experimental+ Structured app context+ <Badge variant="secondary" aria-label="Experimental" className="px-1.5 py-0 text-[10px] font-medium">+ Experimental</Badge></h3><p className="text-xs text-muted-foreground">- Derive compact AI views from the accessibility data Screenpipe already captures. No second recorder or duplicate raw tree.+ Turn what Screenpipe already captures into a clearer view for AI. Nothing is recorded twice.</p></div></div>@@ -3808,10 +3808,10 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre{settings.enableSemanticContext && (<LockedSetting settingKey="semanticContextMode">- <div className="flex items-start justify-between gap-4 border-t border-border pt-3">+ <div className="flex flex-col gap-3 border-t border-border pt-3 sm:flex-row sm:items-start sm:justify-between sm:gap-4"><div className="min-w-0"><label htmlFor="semanticContextMode" className="text-xs font-medium text-foreground">- AI context use+ Use it for</label><p className="text-xs text-muted-foreground mt-0.5">{@@ -3827,8 +3827,8 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. ScrehandleSettingsChange({ semanticContextMode: value }, true)}>- <SelectTrigger id="semanticContextMode" className="w-[180px] h-8 text-xs shrink-0">- <SelectValue />+ <SelectTrigger id="semanticContextMode" className="h-8 w-full shrink-0 text-xs sm:w-[220px]">+ <SelectValue className="min-w-0 flex-1 truncate text-left" /></SelectTrigger><SelectContent><SelectItem value="memory">{SEMANTIC_CONTEXT_MODE_COPY.memory.label}</SelectItem>diff --git a/apps/screenpipe-app-tauri/lib/__tests__/semantic-context-mode.test.ts b/apps/screenpipe-app-tauri/lib/__tests__/semantic-context-mode.test.tsindex 73ef98b8e..f8c88bac3 100644--- a/apps/screenpipe-app-tauri/lib/__tests__/semantic-context-mode.test.ts+++ b/apps/screenpipe-app-tauri/lib/__tests__/semantic-context-mode.test.ts@@ -9,19 +9,21 @@ describe("semantic context mode copy", () => {it("presents memory and automation as the user-facing choices", () => {expect(Object.values(SEMANTIC_CONTEXT_MODE_COPY).map(({ label }) => label),- ).toEqual(["memory", "automation", "memory + automation"]);+ ).toEqual(["Memory", "Automation", "Memory + automation"]);});- it("explains resource and token tradeoffs without renaming persisted values", () => {+ it("explains the choices in plain language", () => {expect(SEMANTIC_CONTEXT_MODE_COPY.memory.description).toContain(- "fewer prompt tokens",+ "information sent to AI",);expect(SEMANTIC_CONTEXT_MODE_COPY.computerUse.description).toContain(- "capture storage",+ "buttons and fields",);- expect(SEMANTIC_CONTEXT_MODE_COPY.both.description).toContain("most CPU");expect(SEMANTIC_CONTEXT_MODE_COPY.both.description).toContain(- "view each agent requests",+ "records everything once",+ );+ expect(SEMANTIC_CONTEXT_MODE_COPY.both.description).toContain(+ "most processing power and storage",);});});diff --git a/apps/screenpipe-app-tauri/lib/semantic-context-mode.ts b/apps/screenpipe-app-tauri/lib/semantic-context-mode.tsindex de99ed95b..532408bb1 100644--- a/apps/screenpipe-app-tauri/lib/semantic-context-mode.ts+++ b/apps/screenpipe-app-tauri/lib/semantic-context-mode.ts@@ -9,18 +9,18 @@ export const SEMANTIC_CONTEXT_MODE_COPY: Record<{ label: string; description: string }> = {memory: {- label: "memory",+ label: "Memory",description:- "For recall, search, summaries, and pipes. Uses fewer prompt tokens for reading; adds a low-priority parser and deduplicated semantic rows.",+ "For finding past work, creating summaries, and running Pipes. This keeps the information sent to AI smaller.",},computerUse: {- label: "automation",+ label: "Automation",description:- "For agents that click and type. Skips the memory parser, but keeps extra controls, state, and bounds, so capture storage and automation prompts are larger.",+ "For AI agents that click and type in apps. This keeps extra details about buttons and fields, so it uses more storage.",},both: {- label: "memory + automation",+ label: "Memory + automation",description:- "One capture feeds both with no duplicate raw tree. This combines both background workloads and uses the most CPU and storage; token use follows the view each agent requests.",+ "For finding past work and letting AI agents click and type. Screenpipe records everything once, but this uses the most processing power and storage.",},};