YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(app): enable structured app context for everyone
louis47504dff789d
diff --git a/apps/screenpipe-app-tauri/components/desktop-remote-control.test.tsx b/apps/screenpipe-app-tauri/components/desktop-remote-control.test.tsxindex 77a48049b..3020ed3f8 100644--- a/apps/screenpipe-app-tauri/components/desktop-remote-control.test.tsx+++ b/apps/screenpipe-app-tauri/components/desktop-remote-control.test.tsx@@ -77,7 +77,7 @@ describe("DesktopRemoteControl", () => {vi.clearAllMocks();mocks.settings = {platform: "macos",- enableSemanticContext: false,+ enableSemanticContext: true,experimentalCoreaudioSystemAudio: true,experimentalMeetingPiggyback: true,filterMusic: true,@@ -129,6 +129,7 @@ describe("DesktopRemoteControl", () => {});it("applies independently targeted defaults with one settings write and restart", async () => {+ mocks.settings.enableSemanticContext = false; // legacy installmocks.settings.experimentalMeetingPiggyback = false; // legacy installmocks.payloads = {"semantic-context-control": {diff --git a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsxindex f8a6418f2..d6823bd30 100644--- a/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx+++ b/apps/screenpipe-app-tauri/components/settings/recording-settings.tsx@@ -11,18 +11,12 @@ import { useEventListener } from "@/lib/hooks/use-event-listener";import { useInterval } from "@/lib/hooks/use-interval";import { useSettingsIndexDriftCheck, type SettingsField } from "./settings-search";import { CaptureFrequencyPreview, AudioCaptureModePreview } from "./setting-previews";-import {- SEMANTIC_CONTEXT_MODE_COPY,- type SemanticContextMode,-} from "@/lib/semantic-context-mode";import {getAecModeSettings,getRemoteAecModePolicy,- getRemoteBooleanPolicy,normalizeAecModeForPlatform,normalizeDesktopRemotePreferences,resolveAecModeRemoteValue,- resolveBooleanRemoteValue,type AecMode,} from "@/lib/desktop-remote-control";import {@@ -49,8 +43,6 @@ export const audioSearchIndex: SettingsField[] = [/** Search fields for the Screen destination. */export const screenSearchIndex: SettingsField[] = [{ label: "Screen context capture", keywords: ["screen", "video", "accessibility"] },- { label: "Structured app context", keywords: ["semantic", "ai", "messages", "email", "tasks", "code"], conditional: true },- { label: "Use it for", keywords: ["memory", "computer use", "automation", "agent", "skills"], conditional: true },{ label: "Screen recording", keywords: ["screenshot", "pixels", "ocr", "jpeg", "capture"] },{ label: "Use all monitors", keywords: ["monitor", "display"], conditional: true },// conditional: monitor picker only renders when "Use all monitors" is off — paired right under that toggle.@@ -2082,11 +2074,6 @@ export function RecordingSettings({ section }: { section: RecordingSettingsSectisettings.aecMode ?? "off",currentPlatform,);- const remoteControlPreferences = normalizeDesktopRemotePreferences(settings);- const semanticContextRemotePolicy = getRemoteBooleanPolicy(- settings,- "semanticContext",- );const aecModeRemotePolicy = getRemoteAecModePolicy(settings);const screenContextEnabled = !settings.disableVision;const screenshotImagesEnabled = screenContextEnabled && !(settings.disableScreenshots ?? false);@@ -3729,97 +3716,6 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre</CardContent></Card>- {!settings.disableVision && (- <Card className="border-border bg-card">- <CardContent className="px-3 py-2.5">- <div className="space-y-3">- <div className="flex items-center justify-between gap-3">- <div className="flex items-center space-x-2.5 min-w-0">- <AppWindowMac className="h-4 w-4 text-muted-foreground shrink-0" />- <div className="min-w-0">- <h3 className="text-sm font-medium text-foreground flex items-center gap-1.5">- Structured app context- <Badge variant="secondary" aria-label="Experimental" className="px-1.5 py-0 text-[10px] font-medium">- Experimental- </Badge>- </h3>- <p className="text-xs text-muted-foreground">- Turn what Screenpipe already captures into a clearer view for AI. Nothing is recorded twice.- </p>- </div>- </div>- {semanticContextRemotePolicy.forceDisabled ? (- <Switch id="enableSemanticContext" checked={false} disabled />- ) : (- <ManagedSwitch- settingKey="enableSemanticContext"- id="enableSemanticContext"- checked={Boolean(settings.enableSemanticContext ?? false)}- onCheckedChange={(checked) =>- handleSettingsChange(- {- remoteControlPreferences: {- ...remoteControlPreferences,- semanticContext: checked,- },- enableSemanticContext: resolveBooleanRemoteValue(- checked,- semanticContextRemotePolicy,- ),- },- true,- )- }- />- )}- </div>-- {semanticContextRemotePolicy.forceDisabled && (- <p className="border-t border-border pt-3 text-xs text-muted-foreground">- Temporarily disabled by the remote safety control. Your- preference is preserved.- </p>- )}-- {settings.enableSemanticContext &&- !semanticContextRemotePolicy.forceDisabled && (- <LockedSetting settingKey="semanticContextMode">- <div className="flex flex-col gap-3 border-t border-border pt-3 sm:flex-row sm:items-start sm:justify-between sm:gap-4">- <div className="min-w-0">- <label htmlFor="semanticContextMode" className="text-xs font-medium text-foreground">- Use it for- </label>- <p className="text-xs text-muted-foreground mt-0.5">- {- SEMANTIC_CONTEXT_MODE_COPY[- (settings.semanticContextMode ?? "memory") as SemanticContextMode- ].description- }- </p>- </div>- <Select- value={settings.semanticContextMode ?? "memory"}- onValueChange={(value: SemanticContextMode) =>- handleSettingsChange({ semanticContextMode: value }, true)- }- >- <SelectTrigger id="semanticContextMode" className="h-8 w-full shrink-0 text-xs sm:w-[220px]">- <SelectValue className="min-w-0 flex-1 truncate text-left" />- </SelectTrigger>- <SelectContent>- <SelectItem value="memory">{SEMANTIC_CONTEXT_MODE_COPY.memory.label}</SelectItem>- <SelectItem value="computerUse">{SEMANTIC_CONTEXT_MODE_COPY.computerUse.label}</SelectItem>- <SelectItem value="both">{SEMANTIC_CONTEXT_MODE_COPY.both.label}</SelectItem>- </SelectContent>- </Select>- </div>- </LockedSetting>- )}- </div>- </CardContent>- </Card>- )}-{!settings.disableVision && (<Card className="border-border bg-card"><CardContent className="px-3 py-2.5">diff --git a/apps/screenpipe-app-tauri/lib/desktop-remote-control.test.ts b/apps/screenpipe-app-tauri/lib/desktop-remote-control.test.tsindex ef523429c..21d3a7aee 100644--- a/apps/screenpipe-app-tauri/lib/desktop-remote-control.test.ts+++ b/apps/screenpipe-app-tauri/lib/desktop-remote-control.test.ts@@ -19,9 +19,41 @@ import {} from "./desktop-remote-control";describe("desktop remote control", () => {+ it.each(["macos", "windows", "linux"])(+ "enables structured context for old opt-outs on %s and honors emergency stops",+ (platform) => {+ const settings = {+ platform,+ enableSemanticContext: false,+ semanticContextPreference: false,+ enterpriseManagedSettings: { enableSemanticContext: false },+ remoteControlPreferences: {+ ...NEW_INSTALL_REMOTE_CONTROL_PREFERENCES,+ semanticContext: false,+ },+ };+ const policy = structuredClone(LOCAL_DESKTOP_REMOTE_POLICY);+ policy.boolean.semanticContext.defaultEnabled = false;+ const enabled = buildDesktopRemoteControlPatch(settings, policy);+ expect(enabled.preferences.semanticContext).toBe(true);+ expect(enabled.patch.enableSemanticContext).toBe(true);+ expect(enabled.recorderRestartRequired).toBe(true);+ policy.boolean.semanticContext.forceDisabled = true;+ const stopped = buildDesktopRemoteControlPatch(+ { ...settings, ...enabled.patch }, policy,+ );+ expect(stopped.patch.enableSemanticContext).toBe(false);+ policy.boolean.semanticContext.forceDisabled = false;+ const restored = buildDesktopRemoteControlPatch(+ { ...settings, ...enabled.patch, ...stopped.patch }, policy,+ );+ expect(restored.patch.enableSemanticContext).toBe(true);+ },+ );+it("preserves each shipped default for missing or malformed flag payloads", () => {expect(parseBooleanRemotePolicy("semanticContext", null)).toEqual({- defaultEnabled: false,+ defaultEnabled: true,forceDisabled: false,});expect(parseBooleanRemotePolicy("coreAudioSystemAudio", undefined)).toEqual(@@ -304,7 +336,7 @@ describe("desktop remote control", () => {remoteControlPreferences: NEW_INSTALL_REMOTE_CONTROL_PREFERENCES,remoteControlPolicy: LOCAL_DESKTOP_REMOTE_POLICY,autoUpdate: false,- enableSemanticContext: false,+ enableSemanticContext: true,experimentalCoreaudioSystemAudio: true,experimentalMeetingPiggyback: true,filterMusic: true,@@ -376,7 +408,7 @@ describe("desktop remote control", () => {expect(buildDesktopRemoteControlPatch(settings, policy).changedControls,- ).toEqual(["smartRecording"]);+ ).toEqual(["semanticContext", "smartRecording"]);});it("ships the rollout capability disabled and never seeds a local opt-in", () => {@@ -423,7 +455,7 @@ describe("desktop remote control", () => {const result = buildDesktopRemoteControlPatch({platform: "macos",- enableSemanticContext: false,+ enableSemanticContext: true,experimentalCoreaudioSystemAudio: true,experimentalMeetingPiggyback: true,filterMusic: true,diff --git a/apps/screenpipe-app-tauri/lib/desktop-remote-control.ts b/apps/screenpipe-app-tauri/lib/desktop-remote-control.tsindex 029a24c1b..b51461715 100644--- a/apps/screenpipe-app-tauri/lib/desktop-remote-control.ts+++ b/apps/screenpipe-app-tauri/lib/desktop-remote-control.ts@@ -15,7 +15,7 @@ export const BOOLEAN_REMOTE_CONTROL_DEFINITIONS = {semanticContext: {flagKey: "semantic-context-control",settingKey: "enableSemanticContext",- shippedDefault: false,+ shippedDefault: true,},coreAudioSystemAudio: {flagKey: "coreaudio-system-audio-control",@@ -144,7 +144,7 @@ export const LOCAL_DESKTOP_REMOTE_POLICY: DesktopRemotePolicySnapshot = {export const NEW_INSTALL_REMOTE_CONTROL_PREFERENCES: DesktopRemotePreferences ={- semanticContext: null,+ semanticContext: true,coreAudioSystemAudio: null,smartRecording: null,filterMusic: null,@@ -469,13 +469,9 @@ export function normalizeDesktopRemotePreferences(settings: RemoteControllableSettings,): DesktopRemotePreferences {const current = settings.remoteControlPreferences;- const legacySemanticPreference = settings.semanticContextPreference;return {- semanticContext: validBooleanPreference(current?.semanticContext)- ? current.semanticContext- : validBooleanPreference(legacySemanticPreference)- ? legacySemanticPreference- : Boolean(settings.enableSemanticContext ?? false),+ // Structured app context is built in, including for former opt-outs.+ semanticContext: true,coreAudioSystemAudio: validBooleanPreference(current?.coreAudioSystemAudio)? current.coreAudioSystemAudio: Boolean(settings.experimentalCoreaudioSystemAudio ?? true),@@ -565,13 +561,8 @@ export function buildDesktopRemoteControlPatch(const preferences = normalizeDesktopRemotePreferences(settings);const managed = settings.enterpriseManagedSettings ?? {};const effective = {- semanticContext: resolveBooleanRemoteControlValue(- "semanticContext",- preferences.semanticContext,- booleanPolicyOf(policy, "semanticContext"),- settings.platform,- parseManagedBoolean(managed.enableSemanticContext),- ),+ // Only the emergency shutoff can disable built-in app context.+ semanticContext: !booleanPolicyOf(policy, "semanticContext").forceDisabled,coreAudioSystemAudio: resolveBooleanRemoteControlValue("coreAudioSystemAudio",preferences.coreAudioSystemAudio,diff --git a/apps/screenpipe-app-tauri/lib/hooks/__tests__/default-settings-remote-controls.test.ts b/apps/screenpipe-app-tauri/lib/hooks/__tests__/default-settings-remote-controls.test.tsindex da20e39b1..4aeceba33 100644--- a/apps/screenpipe-app-tauri/lib/hooks/__tests__/default-settings-remote-controls.test.ts+++ b/apps/screenpipe-app-tauri/lib/hooks/__tests__/default-settings-remote-controls.test.ts@@ -13,7 +13,7 @@ describe("default settings: remote controls", () => {it("ships local behavior while new installs can inherit remote defaults", () => {const settings = createDefaultSettingsObject();- expect(settings.enableSemanticContext).toBe(false);+ expect(settings.enableSemanticContext).toBe(true);expect(settings.experimentalCoreaudioSystemAudio).toBe(true);expect(settings.experimentalMeetingPiggyback).toBe(true);expect(settings.filterMusic).toBe(true);diff --git a/apps/screenpipe-app-tauri/lib/hooks/managed-settings.test.ts b/apps/screenpipe-app-tauri/lib/hooks/managed-settings.test.tsindex fb586d285..8324a0f4b 100644--- a/apps/screenpipe-app-tauri/lib/hooks/managed-settings.test.ts+++ b/apps/screenpipe-app-tauri/lib/hooks/managed-settings.test.ts@@ -15,6 +15,17 @@ import {// is the privacy-critical one — same bug class as the audio fix (#4586).describe("computeManagedSettingUpdates", () => {+ it("ignores the retired structured context toggle in managed settings", () => {+ const current = { enableSemanticContext: true };+ const managed = { enableSemanticContext: "false" };+ expect(computeManagedSettingUpdates(managed, current).engineUpdates)+ .not.toHaveProperty("enableSemanticContext");+ expect(applyManagedOverrides(current, managed).enableSemanticContext).toBe(true);+ expect(applyManagedOverrides(+ { enableSemanticContext: false }, { enableSemanticContext: true },+ ).enableSemanticContext).toBe(false);+ });+it("enforces disableVision (Screen recording: Always off) and flags an engine restart", () => {const r = computeManagedSettingUpdates({ disableVision: "true" }, { disableVision: false });expect(r.engineUpdates.disableVision).toBe(true);diff --git a/apps/screenpipe-app-tauri/lib/hooks/managed-settings.ts b/apps/screenpipe-app-tauri/lib/hooks/managed-settings.tsindex d838c5ef2..e3d76e849 100644--- a/apps/screenpipe-app-tauri/lib/hooks/managed-settings.ts+++ b/apps/screenpipe-app-tauri/lib/hooks/managed-settings.ts@@ -160,7 +160,6 @@ export const MANAGED_SETTING_DEFINITIONS: readonly ManagedSettingDefinition[] =bool("disableVision", false),bool("disableScreenshots", false),- bool("enableSemanticContext", false),enumeration("semanticContextMode", ["memory", "computerUse", "both"], "memory"),bool("disableTimeline", false),bool("useAllMonitors", true),@@ -405,5 +404,8 @@ export function applyManagedOverrides<T extends Record<string, unknown>>(if (!managedValues || typeof managedValues !== "object" || Array.isArray(managedValues)) {return updates;}- return { ...updates, ...(managedValues as Record<string, unknown>) } as T;+ const overrides = { ...(managedValues as Record<string, unknown>) };+ // The retired toggle must not undo built-in context or its emergency shutoff.+ delete overrides.enableSemanticContext;+ return { ...updates, ...overrides } as T;}diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx b/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsxindex 6b72a6fb9..25dc1837b 100644--- a/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx+++ b/apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx@@ -800,7 +800,7 @@ let DEFAULT_SETTINGS: Settings = {lockVaultShortcut: "Super+Shift+L",disableVision: false,disableScreenshots: false,- enableSemanticContext: false,+ enableSemanticContext: true,remoteControlPreferences: {...NEW_INSTALL_REMOTE_CONTROL_PREFERENCES,},diff --git a/apps/screenpipe-app-tauri/src-tauri/src/store.rs b/apps/screenpipe-app-tauri/src-tauri/src/store.rsindex 37f3df814..f7f62885b 100644--- a/apps/screenpipe-app-tauri/src-tauri/src/store.rs+++ b/apps/screenpipe-app-tauri/src-tauri/src/store.rs@@ -1926,7 +1926,7 @@ Rules:max_tokens: 4096,};- // Rust persists store.bin before the frontend mounts. All-null values+ // Rust persists store.bin before the frontend mounts. Null values// identify a genuinely new install that may inherit remote defaults;// legacy stores lack this object and are migrated from their current// effective values. The persisted policy also lets Rust enforce every@@ -1935,7 +1935,7 @@ Rules:("remoteControlPreferences".to_string(),json!({- "semanticContext": null,+ "semanticContext": true,"coreAudioSystemAudio": null,"smartRecording": null,"filterMusic": null,@@ -1949,7 +1949,7 @@ Rules:"schemaVersion": 1,"boolean": {"semanticContext": {- "defaultEnabled": false,+ "defaultEnabled": true,"forceDisabled": false,},"coreAudioSystemAudio": {@@ -1983,6 +1983,7 @@ Rules:Self {// App-specific defaults override RecordingSettings::default() where neededrecording: screenpipe_config::RecordingSettings {+ enable_semantic_context: true,audio_transcription_engine: "whisper-large-v3-turbo-quantized".to_string(),monitor_ids: vec!["default".to_string()],audio_devices: vec!["default".to_string()],@@ -2247,6 +2248,9 @@ impl SettingsStore {pub fn to_recording_settings(&self) -> screenpipe_config::RecordingSettings {let mut settings = self.recording.clone();+ // App context is built in, even for old stores with the toggle off.+ // The remote emergency shutoff below still takes precedence.+ settings.enable_semantic_context = true;// Automatic meeting capture also applies before the frontend mounts,// including old stores with the former opt-in saved as false.settings.experimental_meeting_piggyback =@@ -5240,6 +5244,28 @@ mod tests {assert_eq!(settings.ai_presets.len(), 0);}+ #[test]+ fn structured_context_is_automatic_before_frontend_startup() {+ let mut store = SettingsStore::default();+ assert!(store.recording.enable_semantic_context);+ assert!(store.to_recording_settings().enable_semantic_context);+ store.recording.enable_semantic_context = false;+ store.extra.insert(+ "remoteControlPreferences".into(),+ json!({"semanticContext": false}),+ );+ assert!(store.to_recording_settings().enable_semantic_context);+ store.extra.insert(+ "remoteControlPolicy".into(),+ json!({"schemaVersion": 1, "boolean": {+ "semanticContext": {"defaultEnabled": false, "forceDisabled": true}+ }}),+ );+ assert!(!store.to_recording_settings().enable_semantic_context);+ store.extra.remove("remoteControlPolicy");+ assert!(store.to_recording_settings().enable_semantic_context);+ }+#[test]fn smart_recording_is_automatic_for_legacy_stores_before_frontend_startup() {let mut store = SettingsStore::default();diff --git a/apps/screenpipe-workflows-tauri/components/desktop-remote-control.test.tsx b/apps/screenpipe-workflows-tauri/components/desktop-remote-control.test.tsxindex 76a8ae5ec..04c9bd216 100644--- a/apps/screenpipe-workflows-tauri/components/desktop-remote-control.test.tsx+++ b/apps/screenpipe-workflows-tauri/components/desktop-remote-control.test.tsx@@ -77,7 +77,7 @@ describe("DesktopRemoteControl", () => {vi.clearAllMocks();mocks.settings = {platform: "macos",- enableSemanticContext: false,+ enableSemanticContext: true,experimentalCoreaudioSystemAudio: true,experimentalMeetingPiggyback: false,filterMusic: true,@@ -129,6 +129,7 @@ describe("DesktopRemoteControl", () => {});it("applies independently targeted defaults with one settings write and restart", async () => {+ mocks.settings.enableSemanticContext = false; // legacy installmocks.payloads = {"semantic-context-control": {defaultEnabled: true,diff --git a/apps/screenpipe-workflows-tauri/components/settings/recording-settings.tsx b/apps/screenpipe-workflows-tauri/components/settings/recording-settings.tsxindex baecd4af9..0dc305673 100644--- a/apps/screenpipe-workflows-tauri/components/settings/recording-settings.tsx+++ b/apps/screenpipe-workflows-tauri/components/settings/recording-settings.tsx@@ -11,18 +11,12 @@ import { useEventListener } from "@/lib/hooks/use-event-listener";import { useInterval } from "@/lib/hooks/use-interval";import { useSettingsIndexDriftCheck, type SettingsField } from "./settings-search";import { CaptureFrequencyPreview, AudioCaptureModePreview } from "./setting-previews";-import {- SEMANTIC_CONTEXT_MODE_COPY,- type SemanticContextMode,-} from "@/lib/semantic-context-mode";import {getAecModeSettings,getRemoteAecModePolicy,- getRemoteBooleanPolicy,normalizeAecModeForPlatform,normalizeDesktopRemotePreferences,resolveAecModeRemoteValue,- resolveBooleanRemoteValue,type AecMode,} from "@/lib/desktop-remote-control";import {@@ -50,8 +44,6 @@ export const audioSearchIndex: SettingsField[] = [/** Search fields for the Screen destination. */export const screenSearchIndex: SettingsField[] = [{ label: "Screen context capture", keywords: ["screen", "video", "accessibility"] },- { label: "Structured app context", keywords: ["semantic", "ai", "messages", "email", "tasks", "code"], conditional: true },- { label: "Use it for", keywords: ["memory", "computer use", "automation", "agent", "skills"], conditional: true },{ label: "Screen recording", keywords: ["screenshot", "pixels", "ocr", "jpeg", "capture"] },{ label: "Use all monitors", keywords: ["monitor", "display"], conditional: true },// conditional: monitor picker only renders when "Use all monitors" is off — paired right under that toggle.@@ -2083,11 +2075,6 @@ export function RecordingSettings({ section }: { section: RecordingSettingsSectisettings.aecMode ?? "off",currentPlatform,);- const remoteControlPreferences = normalizeDesktopRemotePreferences(settings);- const semanticContextRemotePolicy = getRemoteBooleanPolicy(- settings,- "semanticContext",- );const smartRecordingRemotePolicy = getRemoteBooleanPolicy(settings,"smartRecording",@@ -3794,97 +3781,6 @@ Your screen is a pipe. Everything you see, hear, and type flows through it. Scre</CardContent></Card>- {!settings.disableVision && (- <Card className="border-border bg-card">- <CardContent className="px-3 py-2.5">- <div className="space-y-3">- <div className="flex items-center justify-between gap-3">- <div className="flex items-center space-x-2.5 min-w-0">- <AppWindowMac className="h-4 w-4 text-muted-foreground shrink-0" />- <div className="min-w-0">- <h3 className="text-sm font-medium text-foreground flex items-center gap-1.5">- Structured app context- <Badge variant="secondary" aria-label="Experimental" className="px-1.5 py-0 text-[10px] font-medium">- Experimental- </Badge>- </h3>- <p className="text-xs text-muted-foreground">- Turn what Screenpipe already captures into a clearer view for AI. Nothing is recorded twice.- </p>- </div>- </div>- {semanticContextRemotePolicy.forceDisabled ? (- <Switch id="enableSemanticContext" checked={false} disabled />- ) : (- <ManagedSwitch- settingKey="enableSemanticContext"- id="enableSemanticContext"- checked={Boolean(settings.enableSemanticContext ?? false)}- onCheckedChange={(checked) =>- handleSettingsChange(- {- remoteControlPreferences: {- ...remoteControlPreferences,- semanticContext: checked,- },- enableSemanticContext: resolveBooleanRemoteValue(- checked,- semanticContextRemotePolicy,- ),- },- true,- )- }- />- )}- </div>-- {semanticContextRemotePolicy.forceDisabled && (- <p className="border-t border-border pt-3 text-xs text-muted-foreground">- Temporarily disabled by the remote safety control. Your- preference is preserved.- </p>- )}-- {settings.enableSemanticContext &&- !semanticContextRemotePolicy.forceDisabled && (- <LockedSetting settingKey="semanticContextMode">- <div className="flex flex-col gap-3 border-t border-border pt-3 sm:flex-row sm:items-start sm:justify-between sm:gap-4">- <div className="min-w-0">- <label htmlFor="semanticContextMode" className="text-xs font-medium text-foreground">- Use it for- </label>- <p className="text-xs text-muted-foreground mt-0.5">- {- SEMANTIC_CONTEXT_MODE_COPY[- (settings.semanticContextMode ?? "memory") as SemanticContextMode- ].description- }- </p>- </div>- <Select- value={settings.semanticContextMode ?? "memory"}- onValueChange={(value: SemanticContextMode) =>- handleSettingsChange({ semanticContextMode: value }, true)- }- >- <SelectTrigger id="semanticContextMode" className="h-8 w-full shrink-0 text-xs sm:w-[220px]">- <SelectValue className="min-w-0 flex-1 truncate text-left" />- </SelectTrigger>- <SelectContent>- <SelectItem value="memory">{SEMANTIC_CONTEXT_MODE_COPY.memory.label}</SelectItem>- <SelectItem value="computerUse">{SEMANTIC_CONTEXT_MODE_COPY.computerUse.label}</SelectItem>- <SelectItem value="both">{SEMANTIC_CONTEXT_MODE_COPY.both.label}</SelectItem>- </SelectContent>- </Select>- </div>- </LockedSetting>- )}- </div>- </CardContent>- </Card>- )}-{!settings.disableVision && (<Card className="border-border bg-card"><CardContent className="px-3 py-2.5">diff --git a/apps/screenpipe-workflows-tauri/lib/desktop-remote-control.test.ts b/apps/screenpipe-workflows-tauri/lib/desktop-remote-control.test.tsindex 32e814487..3504c6ecc 100644--- a/apps/screenpipe-workflows-tauri/lib/desktop-remote-control.test.ts+++ b/apps/screenpipe-workflows-tauri/lib/desktop-remote-control.test.ts@@ -19,9 +19,41 @@ import {} from "./desktop-remote-control";describe("desktop remote control", () => {+ it.each(["macos", "windows", "linux"])(+ "enables structured context for old opt-outs on %s and honors emergency stops",+ (platform) => {+ const settings = {+ platform,+ enableSemanticContext: false,+ semanticContextPreference: false,+ enterpriseManagedSettings: { enableSemanticContext: false },+ remoteControlPreferences: {+ ...NEW_INSTALL_REMOTE_CONTROL_PREFERENCES,+ semanticContext: false,+ },+ };+ const policy = structuredClone(LOCAL_DESKTOP_REMOTE_POLICY);+ policy.boolean.semanticContext.defaultEnabled = false;+ const enabled = buildDesktopRemoteControlPatch(settings, policy);+ expect(enabled.preferences.semanticContext).toBe(true);+ expect(enabled.patch.enableSemanticContext).toBe(true);+ expect(enabled.recorderRestartRequired).toBe(true);+ policy.boolean.semanticContext.forceDisabled = true;+ const stopped = buildDesktopRemoteControlPatch(+ { ...settings, ...enabled.patch }, policy,+ );+ expect(stopped.patch.enableSemanticContext).toBe(false);+ policy.boolean.semanticContext.forceDisabled = false;+ const restored = buildDesktopRemoteControlPatch(+ { ...settings, ...enabled.patch, ...stopped.patch }, policy,+ );+ expect(restored.patch.enableSemanticContext).toBe(true);+ },+ );+it("preserves each shipped default for missing or malformed flag payloads", () => {expect(parseBooleanRemotePolicy("semanticContext", null)).toEqual({- defaultEnabled: false,+ defaultEnabled: true,forceDisabled: false,});expect(parseBooleanRemotePolicy("coreAudioSystemAudio", undefined)).toEqual(@@ -281,7 +313,7 @@ describe("desktop remote control", () => {remoteControlPreferences: NEW_INSTALL_REMOTE_CONTROL_PREFERENCES,remoteControlPolicy: LOCAL_DESKTOP_REMOTE_POLICY,autoUpdate: false,- enableSemanticContext: false,+ enableSemanticContext: true,experimentalCoreaudioSystemAudio: true,experimentalMeetingPiggyback: false,filterMusic: true,@@ -353,7 +385,7 @@ describe("desktop remote control", () => {expect(buildDesktopRemoteControlPatch(settings, policy).changedControls,- ).toEqual([]);+ ).toEqual(["semanticContext"]);});it("ships the rollout capability disabled and never seeds a local opt-in", () => {@@ -400,7 +432,7 @@ describe("desktop remote control", () => {const result = buildDesktopRemoteControlPatch({platform: "macos",- enableSemanticContext: false,+ enableSemanticContext: true,experimentalCoreaudioSystemAudio: true,experimentalMeetingPiggyback: false,filterMusic: true,diff --git a/apps/screenpipe-workflows-tauri/lib/desktop-remote-control.ts b/apps/screenpipe-workflows-tauri/lib/desktop-remote-control.tsindex 2102599c0..a4db550de 100644--- a/apps/screenpipe-workflows-tauri/lib/desktop-remote-control.ts+++ b/apps/screenpipe-workflows-tauri/lib/desktop-remote-control.ts@@ -15,7 +15,7 @@ export const BOOLEAN_REMOTE_CONTROL_DEFINITIONS = {semanticContext: {flagKey: "semantic-context-control",settingKey: "enableSemanticContext",- shippedDefault: false,+ shippedDefault: true,},coreAudioSystemAudio: {flagKey: "coreaudio-system-audio-control",@@ -144,7 +144,7 @@ export const LOCAL_DESKTOP_REMOTE_POLICY: DesktopRemotePolicySnapshot = {export const NEW_INSTALL_REMOTE_CONTROL_PREFERENCES: DesktopRemotePreferences ={- semanticContext: null,+ semanticContext: true,coreAudioSystemAudio: null,smartRecording: null,filterMusic: null,@@ -465,13 +465,9 @@ export function normalizeDesktopRemotePreferences(settings: RemoteControllableSettings,): DesktopRemotePreferences {const current = settings.remoteControlPreferences;- const legacySemanticPreference = settings.semanticContextPreference;return {- semanticContext: validBooleanPreference(current?.semanticContext)- ? current.semanticContext- : validBooleanPreference(legacySemanticPreference)- ? legacySemanticPreference- : Boolean(settings.enableSemanticContext ?? false),+ // Structured app context is built in, including for former opt-outs.+ semanticContext: true,coreAudioSystemAudio: validBooleanPreference(current?.coreAudioSystemAudio)? current.coreAudioSystemAudio: Boolean(settings.experimentalCoreaudioSystemAudio ?? true),@@ -563,13 +559,8 @@ export function buildDesktopRemoteControlPatch(const preferences = normalizeDesktopRemotePreferences(settings);const managed = settings.enterpriseManagedSettings ?? {};const effective = {- semanticContext: resolveBooleanRemoteControlValue(- "semanticContext",- preferences.semanticContext,- booleanPolicyOf(policy, "semanticContext"),- settings.platform,- parseManagedBoolean(managed.enableSemanticContext),- ),+ // Only the emergency shutoff can disable built-in app context.+ semanticContext: !booleanPolicyOf(policy, "semanticContext").forceDisabled,coreAudioSystemAudio: resolveBooleanRemoteControlValue("coreAudioSystemAudio",preferences.coreAudioSystemAudio,diff --git a/apps/screenpipe-workflows-tauri/lib/hooks/__tests__/default-settings-remote-controls.test.ts b/apps/screenpipe-workflows-tauri/lib/hooks/__tests__/default-settings-remote-controls.test.tsindex 5e032b8d2..53763a63c 100644--- a/apps/screenpipe-workflows-tauri/lib/hooks/__tests__/default-settings-remote-controls.test.ts+++ b/apps/screenpipe-workflows-tauri/lib/hooks/__tests__/default-settings-remote-controls.test.ts@@ -13,7 +13,7 @@ describe("default settings: remote controls", () => {it("ships local behavior while new installs can inherit remote defaults", () => {const settings = createDefaultSettingsObject();- expect(settings.enableSemanticContext).toBe(false);+ expect(settings.enableSemanticContext).toBe(true);expect(settings.experimentalCoreaudioSystemAudio).toBe(true);expect(settings.experimentalMeetingPiggyback).toBe(false);expect(settings.filterMusic).toBe(true);diff --git a/apps/screenpipe-workflows-tauri/lib/hooks/managed-settings.test.ts b/apps/screenpipe-workflows-tauri/lib/hooks/managed-settings.test.tsindex fb586d285..8324a0f4b 100644--- a/apps/screenpipe-workflows-tauri/lib/hooks/managed-settings.test.ts+++ b/apps/screenpipe-workflows-tauri/lib/hooks/managed-settings.test.ts@@ -15,6 +15,17 @@ import {// is the privacy-critical one — same bug class as the audio fix (#4586).describe("computeManagedSettingUpdates", () => {+ it("ignores the retired structured context toggle in managed settings", () => {+ const current = { enableSemanticContext: true };+ const managed = { enableSemanticContext: "false" };+ expect(computeManagedSettingUpdates(managed, current).engineUpdates)+ .not.toHaveProperty("enableSemanticContext");+ expect(applyManagedOverrides(current, managed).enableSemanticContext).toBe(true);+ expect(applyManagedOverrides(+ { enableSemanticContext: false }, { enableSemanticContext: true },+ ).enableSemanticContext).toBe(false);+ });+it("enforces disableVision (Screen recording: Always off) and flags an engine restart", () => {const r = computeManagedSettingUpdates({ disableVision: "true" }, { disableVision: false });expect(r.engineUpdates.disableVision).toBe(true);diff --git a/apps/screenpipe-workflows-tauri/lib/hooks/managed-settings.ts b/apps/screenpipe-workflows-tauri/lib/hooks/managed-settings.tsindex 6927c444b..4d5125eb4 100644--- a/apps/screenpipe-workflows-tauri/lib/hooks/managed-settings.ts+++ b/apps/screenpipe-workflows-tauri/lib/hooks/managed-settings.ts@@ -161,7 +161,6 @@ export const MANAGED_SETTING_DEFINITIONS: readonly ManagedSettingDefinition[] =bool("disableVision", false),bool("disableScreenshots", false),- bool("enableSemanticContext", false),enumeration("semanticContextMode", ["memory", "computerUse", "both"], "memory"),bool("disableTimeline", false),bool("useAllMonitors", true),@@ -406,5 +405,8 @@ export function applyManagedOverrides<T extends Record<string, unknown>>(if (!managedValues || typeof managedValues !== "object" || Array.isArray(managedValues)) {return updates;}- return { ...updates, ...(managedValues as Record<string, unknown>) } as T;+ const overrides = { ...(managedValues as Record<string, unknown>) };+ // The retired toggle must not undo built-in context or its emergency shutoff.+ delete overrides.enableSemanticContext;+ return { ...updates, ...overrides } as T;}diff --git a/apps/screenpipe-workflows-tauri/lib/hooks/use-settings.tsx b/apps/screenpipe-workflows-tauri/lib/hooks/use-settings.tsxindex d5a1959d7..1e5211b29 100644--- a/apps/screenpipe-workflows-tauri/lib/hooks/use-settings.tsx+++ b/apps/screenpipe-workflows-tauri/lib/hooks/use-settings.tsx@@ -768,7 +768,7 @@ let DEFAULT_SETTINGS: Settings = {lockVaultShortcut: "Super+Shift+L",disableVision: false,disableScreenshots: false,- enableSemanticContext: false,+ enableSemanticContext: true,remoteControlPreferences: {...NEW_INSTALL_REMOTE_CONTROL_PREFERENCES,},diff --git a/apps/screenpipe-workflows-tauri/src-tauri/src/store.rs b/apps/screenpipe-workflows-tauri/src-tauri/src/store.rsindex 372749b29..12848f2c8 100644--- a/apps/screenpipe-workflows-tauri/src-tauri/src/store.rs+++ b/apps/screenpipe-workflows-tauri/src-tauri/src/store.rs@@ -1764,7 +1764,7 @@ Rules:max_tokens: 4096,};- // Rust persists store.bin before the frontend mounts. All-null values+ // Rust persists store.bin before the frontend mounts. Null values// identify a genuinely new install that may inherit remote defaults;// legacy stores lack this object and are migrated from their current// effective values. The persisted policy also lets Rust enforce every@@ -1773,7 +1773,7 @@ Rules:("remoteControlPreferences".to_string(),json!({- "semanticContext": null,+ "semanticContext": true,"coreAudioSystemAudio": null,"smartRecording": null,"filterMusic": null,@@ -1787,7 +1787,7 @@ Rules:"schemaVersion": 1,"boolean": {"semanticContext": {- "defaultEnabled": false,+ "defaultEnabled": true,"forceDisabled": false,},"coreAudioSystemAudio": {@@ -1821,6 +1821,7 @@ Rules:Self {// App-specific defaults override RecordingSettings::default() where neededrecording: screenpipe_config::RecordingSettings {+ enable_semantic_context: true,audio_transcription_engine: "whisper-large-v3-turbo-quantized".to_string(),monitor_ids: vec!["default".to_string()],audio_devices: vec!["default".to_string()],@@ -2085,6 +2086,9 @@ impl SettingsStore {pub fn to_recording_settings(&self) -> screenpipe_config::RecordingSettings {let mut settings = self.recording.clone();+ // App context is built in, even for old stores with the toggle off.+ // The remote emergency shutoff below still takes precedence.+ settings.enable_semantic_context = true;// Override user_id with the Clerk JWT token from the auth user object.// This token is used as the Bearer credential for screenpipe cloud// (transcription proxy, Pi agent, etc.), not as a database ID.@@ -4883,6 +4887,28 @@ mod tests {assert_eq!(settings.ai_presets.len(), 0);}+ #[test]+ fn structured_context_is_automatic_before_frontend_startup() {+ let mut store = SettingsStore::default();+ assert!(store.recording.enable_semantic_context);+ assert!(store.to_recording_settings().enable_semantic_context);+ store.recording.enable_semantic_context = false;+ store.extra.insert(+ "remoteControlPreferences".into(),+ json!({"semanticContext": false}),+ );+ assert!(store.to_recording_settings().enable_semantic_context);+ store.extra.insert(+ "remoteControlPolicy".into(),+ json!({"schemaVersion": 1, "boolean": {+ "semanticContext": {"defaultEnabled": false, "forceDisabled": true}+ }}),+ );+ assert!(!store.to_recording_settings().enable_semantic_context);+ store.extra.remove("remoteControlPolicy");+ assert!(store.to_recording_settings().enable_semantic_context);+ }+#[test]fn remote_force_offs_win_after_recording_settings() {let mut store = SettingsStore::default();