Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(app): polish structured context setting (#5534)

Louis Beaumont55daa12869e2
diff --git a/.github/pr-media/structured-app-context-setting.png b/.github/pr-media/structured-app-context-setting.png
new file mode 100644
index 000000000..e98d3b28b
Binary files /dev/null and b/.github/pr-media/structured-app-context-setting.png differ
diff --git a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx
index c65d6cd5e..98fbd821b 100644
--- a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx
@@ -40,8 +40,8 @@ export const searchIndex: SettingsField[] = [
{ label: "Smart recording", keywords: ["smart recording", "beta", "meeting", "piggyback", "per-process", "meeting audio"], conditional: true },
{ label: "Bluetooth microphones", keywords: ["bluetooth", "airpods", "headset", "a2dp", "sco", "meeting"], conditional: true },
{ label: "Screen context capture", keywords: ["screen", "video", "accessibility"] },
- { label: "structured app context", keywords: ["semantic", "ai", "messages", "email", "tasks", "code"], conditional: true },
- { label: "AI context use", keywords: ["memory", "computer use", "automation", "agent", "skills"], conditional: true },
+ { label: "Structured app context", keywords: ["semantic", "ai", "messages", "email", "tasks", "code"], conditional: true },
+ { label: "Use it for", keywords: ["memory", "computer use", "automation", "agent", "skills"], conditional: true },
{ label: "Screenshot images", keywords: ["screenshot", "pixels", "ocr", "jpeg"] },
{ label: "Use all monitors", keywords: ["monitor", "display"], conditional: true },
// conditional: monitor picker only renders when "Use all monitors" is off — paired right under that toggle.
@@ -3788,13 +3788,13 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre
<AppWindowMac className="h-4 w-4 text-muted-foreground shrink-0" />
<div className="min-w-0">
<h3 className="text-sm font-medium text-foreground flex items-center gap-1.5">
- structured app context
- <Badge variant="secondary" aria-label="experimental" className="px-1.5 py-0 text-[10px] font-medium uppercase tracking-wide">
- experimental
+ Structured app context
+ <Badge variant="secondary" aria-label="Experimental" className="px-1.5 py-0 text-[10px] font-medium">
+ Experimental
</Badge>
</h3>
<p className="text-xs text-muted-foreground">
- Derive compact AI views from the accessibility data Screenpipe already captures. No second recorder or duplicate raw tree.
+ Turn what Screenpipe already captures into a clearer view for AI. Nothing is recorded twice.
</p>
</div>
</div>
@@ -3808,10 +3808,10 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre
{settings.enableSemanticContext && (
<LockedSetting settingKey="semanticContextMode">
- <div className="flex items-start justify-between gap-4 border-t border-border pt-3">
+ <div className="flex flex-col gap-3 border-t border-border pt-3 sm:flex-row sm:items-start sm:justify-between sm:gap-4">
<div className="min-w-0">
<label htmlFor="semanticContextMode" className="text-xs font-medium text-foreground">
- AI context use
+ Use it for
</label>
<p className="text-xs text-muted-foreground mt-0.5">
{
@@ -3827,8 +3827,8 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre
handleSettingsChange({ semanticContextMode: value }, true)
}
>
- <SelectTrigger id="semanticContextMode" className="w-[180px] h-8 text-xs shrink-0">
- <SelectValue />
+ <SelectTrigger id="semanticContextMode" className="h-8 w-full shrink-0 text-xs sm:w-[220px]">
+ <SelectValue className="min-w-0 flex-1 truncate text-left" />
</SelectTrigger>
<SelectContent>
<SelectItem value="memory">{SEMANTIC_CONTEXT_MODE_COPY.memory.label}</SelectItem>
diff --git a/apps/screenpipe-app-tauri/lib/__tests__/semantic-context-mode.test.ts b/apps/screenpipe-app-tauri/lib/__tests__/semantic-context-mode.test.ts
index 73ef98b8e..f8c88bac3 100644
--- a/apps/screenpipe-app-tauri/lib/__tests__/semantic-context-mode.test.ts
+++ b/apps/screenpipe-app-tauri/lib/__tests__/semantic-context-mode.test.ts
@@ -9,19 +9,21 @@ describe("semantic context mode copy", () => {
it("presents memory and automation as the user-facing choices", () => {
expect(
Object.values(SEMANTIC_CONTEXT_MODE_COPY).map(({ label }) => label),
- ).toEqual(["memory", "automation", "memory + automation"]);
+ ).toEqual(["Memory", "Automation", "Memory + automation"]);
});
- it("explains resource and token tradeoffs without renaming persisted values", () => {
+ it("explains the choices in plain language", () => {
expect(SEMANTIC_CONTEXT_MODE_COPY.memory.description).toContain(
- "fewer prompt tokens",
+ "information sent to AI",
);
expect(SEMANTIC_CONTEXT_MODE_COPY.computerUse.description).toContain(
- "capture storage",
+ "buttons and fields",
);
- expect(SEMANTIC_CONTEXT_MODE_COPY.both.description).toContain("most CPU");
expect(SEMANTIC_CONTEXT_MODE_COPY.both.description).toContain(
- "view each agent requests",
+ "records everything once",
+ );
+ expect(SEMANTIC_CONTEXT_MODE_COPY.both.description).toContain(
+ "most processing power and storage",
);
});
});
diff --git a/apps/screenpipe-app-tauri/lib/semantic-context-mode.ts b/apps/screenpipe-app-tauri/lib/semantic-context-mode.ts
index de99ed95b..532408bb1 100644
--- a/apps/screenpipe-app-tauri/lib/semantic-context-mode.ts
+++ b/apps/screenpipe-app-tauri/lib/semantic-context-mode.ts
@@ -9,18 +9,18 @@ export const SEMANTIC_CONTEXT_MODE_COPY: Record<
{ label: string; description: string }
> = {
memory: {
- label: "memory",
+ label: "Memory",
description:
- "For recall, search, summaries, and pipes. Uses fewer prompt tokens for reading; adds a low-priority parser and deduplicated semantic rows.",
+ "For finding past work, creating summaries, and running Pipes. This keeps the information sent to AI smaller.",
},
computerUse: {
- label: "automation",
+ label: "Automation",
description:
- "For agents that click and type. Skips the memory parser, but keeps extra controls, state, and bounds, so capture storage and automation prompts are larger.",
+ "For AI agents that click and type in apps. This keeps extra details about buttons and fields, so it uses more storage.",
},
both: {
- label: "memory + automation",
+ label: "Memory + automation",
description:
- "One capture feeds both with no duplicate raw tree. This combines both background workloads and uses the most CPU and storage; token use follows the view each agent requests.",
+ "For finding past work and letting AI agents click and type. Screenpipe records everything once, but this uses the most processing power and storage.",
},
};