YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
ix(chat): show device connection suggestions in composer (#4072)
Ansh Grover2778aae64374
diff --git a/apps/screenpipe-app-tauri/components/settings/connections-section.tsx b/apps/screenpipe-app-tauri/components/settings/connections-section.tsxindex b3a3ec7aa..5407c5da7 100644--- a/apps/screenpipe-app-tauri/components/settings/connections-section.tsx+++ b/apps/screenpipe-app-tauri/components/settings/connections-section.tsx@@ -19,7 +19,14 @@ import { useSettings } from "@/lib/hooks/use-settings";import { useToast } from "@/components/ui/use-toast";import { ensureChatGptPreset } from "@/lib/utils/chatgpt-preset";import { notifyConnectionsUpdated } from "@/lib/connections-events";-import { CONNECTION_CATEGORY_BY_ID } from "@/lib/constants/connections";+import {+ CONNECTION_CATEGORY_BY_ID,+ CONNECTION_HARDCODED_DESCRIPTIONS,+ compareConnectionTiles,+ getSuggestedConnectionsForDevice,+ normalizeConnectionCategory,+ type ConnectionSuggestionTile,+} from "@/lib/constants/connections";import { Command } from "@tauri-apps/plugin-shell";import { openUrl } from "@tauri-apps/plugin-opener";import { message, open as openDialog } from "@tauri-apps/plugin-dialog";@@ -781,131 +788,12 @@ export function IntegrationIcon({// Connection tile (compact grid item)// ----------------------------------------------------------------------------interface ConnectionTile {- id: string;- name: string;- icon: string;- connected: boolean;- detected?: boolean;- category?: string;- description?: string;-}--const HARDCODED_DESCRIPTIONS: Record<string, string> = {- "claude": "Search your screen & audio from Claude Desktop via MCP",- "cursor": "Give Cursor AI access to your screen history via MCP",- "codex": "Give Codex access to your screen & audio via MCP",- "claude-code": "Add screen memory to the Claude Code CLI",- "warp": "Search screen history from Warp terminal via MCP",- "chatgpt": "Search your screen history from ChatGPT",- "browser-url": "Capture visited URLs from your browser in real time",- "voice-memos": "Sync Apple Voice Memos for AI-powered search",- "apple-intelligence": "Connect Apple Intelligence writing tools",- "input-monitoring": "Track keyboard & mouse for productivity insights",- "apple-calendar": "Search Apple Calendar events with AI",- "google-calendar": "Search Google Calendar events with AI",- "google-docs": "Read and search your Google Docs",- "google-sheets": "Read and search your Google Sheets",- "gmail": "Read and search your Gmail inbox",- "ics-calendar": "Subscribe to any ICS calendar feed",- "openclaw": "Browse the web with OpenClaw agents",- "hermes": "AI-powered messaging assistant",- "whatsapp": "Search your WhatsApp conversations",- "anythingllm": "Give AnythingLLM access to your screen",- "ollama": "Connect local Ollama models to screenpipe",- "lmstudio": "Connect LM Studio models to screenpipe",- "msty": "Connect Msty models to screenpipe",- "obsidian": "Sync screen memory to your Obsidian vault",- "notion": "Search Notion pages with your screen context",- "linear": "Search Linear issues from your screen context",- "perplexity": "Search the web with Perplexity AI",- "krisp": "Search Krisp meeting transcripts and notes",- "plaud": "Search Plaud recordings and transcripts",- "excalidraw": "Search and edit your Excalidraw+ whiteboards",- "custom-mcp": "Connect any MCP-compatible server",- "skills": "Import Claude Code skills for AI automations",-};+type ConnectionTile = ConnectionSuggestionTile & { icon: string };type ConnectionSort = "suggested" | "alphabetical";const ALL_CONNECTION_CATEGORIES = "All";-// High-activation defaults fill the suggested row when there are not enough-// detected or already-connected apps on the device.-const FEATURED_CONNECTION_IDS = [- "custom-mcp",- "claude",- "cursor",- "codex",- "claude-code",- "chatgpt",- "slack",- "obsidian",- "notion",-];---const DEVICE_CONNECTION_ORDER = [- "custom-mcp",- "claude",- "cursor",- "codex",- "claude-code",- "chatgpt",- "browser-url",- "input-monitoring",- "obsidian",- "notion",- "linear",- "slack",- "gmail",- "apple-calendar",- "google-calendar",- "google-docs",- "google-sheets",- "warp",- "ollama",- "lmstudio",- "msty",- "krisp",- "whatsapp",-];--function normalizeConnectionCategory(category: string | null | undefined): string {- const value = (category || "Other").trim();- if (!value) return "Other";- return value- .split(/[-_\s]+/)- .filter(Boolean)- // Preserve all-caps acronyms (AI, CRM) instead of mangling them to "Ai"/"Crm".- .map((part) =>- /^[A-Z0-9]{2,}$/.test(part)- ? part- : part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()- )- .join(" ");-}--function connectionPriority(tile: ConnectionTile): number {- if (tile.connected) return 0;- if (tile.detected) return 1;- if (FEATURED_CONNECTION_IDS.includes(tile.id)) return 2;- return 3;-}--function connectionOrder(tile: ConnectionTile): number {- const index = DEVICE_CONNECTION_ORDER.indexOf(tile.id);- return index === -1 ? DEVICE_CONNECTION_ORDER.length : index;-}--function compareConnectionTiles(a: ConnectionTile, b: ConnectionTile): number {- const priority = connectionPriority(a) - connectionPriority(b);- if (priority !== 0) return priority;- const order = connectionOrder(a) - connectionOrder(b);- if (order !== 0) return order;- return a.name.localeCompare(b.name);-}-// Per-connection quickstart prompts shown when "Try in Chat" is clicked.export const TRY_IN_CHAT_PROMPTS: Record<string, string> = {@@ -3627,7 +3515,7 @@ export function ConnectionsSection({...tile,// Our explicit map overrides the API's category so known tools always land in the right groupcategory: CONNECTION_CATEGORY_BY_ID[tile.id] ?? tile.category ?? "Other",- description: tile.description ?? HARDCODED_DESCRIPTIONS[tile.id],+ description: tile.description ?? CONNECTION_HARDCODED_DESCRIPTIONS[tile.id],}));}, [os, claudeInstalled, cursorInstalled, codexInstalled, chatgptConnected, browserUrlConnected, browserUrlDetected, integrations, appleCalendarConnected, googleCalendarConnected, googleDocsConnected, googleSheetsConnected, gmailConnected, customMcpConnected, customMcpServerCount, krispConnected, plaudConnected, excalidrawConnected, inputMonitoringGranted, importedSkillsCount, detectedConnectionIds]);@@ -3635,14 +3523,7 @@ export function ConnectionsSection({const suggested = useMemo(() => {if (!isDefaultView) return [];- return [...allTiles]- .filter((tile) => (- tile.connected ||- tile.detected ||- FEATURED_CONNECTION_IDS.includes(tile.id)- ))- .sort(compareConnectionTiles)- .slice(0, 8);+ return getSuggestedConnectionsForDevice(allTiles, 8);}, [allTiles, isDefaultView]);// Flat search results (used when search is active or category is programmatically focused)diff --git a/apps/screenpipe-app-tauri/components/standalone-chat.tsx b/apps/screenpipe-app-tauri/components/standalone-chat.tsxindex 19f417e1b..dca812fdf 100644--- a/apps/screenpipe-app-tauri/components/standalone-chat.tsx+++ b/apps/screenpipe-app-tauri/components/standalone-chat.tsx@@ -67,6 +67,7 @@ import { buildChipModelContent, buildChipDisplayContent, parseConnectionChip } fimport { getCurrentWindow } from "@tauri-apps/api/window";import { getCurrentWebview } from "@tauri-apps/api/webview";import { usePlatform } from "@/lib/hooks/use-platform";+import { useHardcodedTiles } from "@/lib/hooks/use-hardcoded-tiles";import { useIsFullscreen } from "@/lib/hooks/use-is-fullscreen";import { useChatFilePreview } from "@/lib/hooks/use-chat-file-preview";import { useSqlAutocomplete, useTagAutocomplete } from "@/lib/hooks/use-sql-autocomplete";@@ -119,6 +120,12 @@ import {import { usePipes } from "@/lib/hooks/use-pipes";import { localFetch, getApiBaseUrl } from "@/lib/api";import { CONNECTIONS_UPDATED_EVENT } from "@/lib/connections-events";+import {+ CONNECTION_CATEGORY_BY_ID,+ CONNECTION_HARDCODED_DESCRIPTIONS,+ getSuggestedConnectionsForDevice,+ normalizeConnectionCategory,+} from "@/lib/constants/connections";import {computeChatCitationPlan,formatSourceCitationsMarkdown,@@ -2504,6 +2511,7 @@ export function StandaloneChat({} = {}) {const { settings, updateSettings, isSettingsLoaded, reloadStore } = useSettings();const { isMac, isWindows, isLoading: isPlatformLoading } = usePlatform();+ const hardcodedConnectionTiles = useHardcodedTiles();// Drop the macOS traffic-light reservation when the window is fullscreen// (the buttons hide). Only relevant in standalone mode (no parent// className) — the embedded variant is below the host's chrome anyway.@@ -2526,6 +2534,30 @@ export function StandaloneChat({() => buildConnectionSetupSuggestions(allConnectionItems, appItems),[allConnectionItems, appItems]);+ const suggestedConnectionTiles = React.useMemo(() => {+ const apiById = new Map(allConnectionItems.map((connection) => [connection.id, connection]));+ const hardcodedIds = new Set(hardcodedConnectionTiles.map((connection) => connection.id));+ const hardcodedTiles = hardcodedConnectionTiles.map((connection) => {+ const apiConnection = apiById.get(connection.id);+ return {+ ...connection,+ icon: connection.icon || apiConnection?.icon || connection.id,+ connected: apiConnection?.connected ?? connection.connected,+ category: CONNECTION_CATEGORY_BY_ID[connection.id] ?? normalizeConnectionCategory(apiConnection?.category),+ description: apiConnection?.description ?? CONNECTION_HARDCODED_DESCRIPTIONS[connection.id],+ };+ });+ const apiTiles = allConnectionItems+ .filter((connection) => !hardcodedIds.has(connection.id) && connection.id !== "owned-default")+ .map((connection) => ({+ ...connection,+ icon: connection.icon || connection.id,+ category: CONNECTION_CATEGORY_BY_ID[connection.id] ?? normalizeConnectionCategory(connection.category),+ description: connection.description ?? CONNECTION_HARDCODED_DESCRIPTIONS[connection.id],+ }));++ return getSuggestedConnectionsForDevice([...hardcodedTiles, ...apiTiles], 8);+ }, [allConnectionItems, hardcodedConnectionTiles]);const refreshConnectionState = React.useCallback(async () => {if (isPlatformLoading) return;try {@@ -9582,9 +9614,7 @@ export function StandaloneChat({Connect your apps to get better answers</button><div className="flex items-center gap-1">- {connections- .filter((c) => INTEGRATION_ICON_KEYS.has(c.icon || c.id))- .slice(0, 8)+ {suggestedConnectionTiles.map((c) => (<buttonkey={c.id}diff --git a/apps/screenpipe-app-tauri/lib/constants/connections.ts b/apps/screenpipe-app-tauri/lib/constants/connections.tsindex e3e4455d1..a34dfd07b 100644--- a/apps/screenpipe-app-tauri/lib/constants/connections.ts+++ b/apps/screenpipe-app-tauri/lib/constants/connections.ts@@ -141,3 +141,140 @@ export const CONNECTION_CATEGORY_BY_ID: Record<string, string> = {"apple-intelligence": "System","voice-memos": "System",};++export interface ConnectionSuggestionTile {+ id: string;+ name: string;+ icon?: string;+ connected: boolean;+ detected?: boolean;+ category?: string;+ description?: string;+}++export const CONNECTION_HARDCODED_DESCRIPTIONS: Record<string, string> = {+ "claude": "Search your screen & audio from Claude Desktop via MCP",+ "cursor": "Give Cursor AI access to your screen history via MCP",+ "codex": "Give Codex access to your screen & audio via MCP",+ "claude-code": "Add screen memory to the Claude Code CLI",+ "warp": "Search screen history from Warp terminal via MCP",+ "chatgpt": "Search your screen history from ChatGPT",+ "browser-url": "Capture visited URLs from your browser in real time",+ "voice-memos": "Sync Apple Voice Memos for AI-powered search",+ "apple-intelligence": "Connect Apple Intelligence writing tools",+ "input-monitoring": "Track keyboard & mouse for productivity insights",+ "apple-calendar": "Search Apple Calendar events with AI",+ "google-calendar": "Search Google Calendar events with AI",+ "google-docs": "Read and search your Google Docs",+ "google-sheets": "Read and search your Google Sheets",+ "gmail": "Read and search your Gmail inbox",+ "ics-calendar": "Subscribe to any ICS calendar feed",+ "openclaw": "Browse the web with OpenClaw agents",+ "hermes": "AI-powered messaging assistant",+ "whatsapp": "Search your WhatsApp conversations",+ "anythingllm": "Give AnythingLLM access to your screen",+ "ollama": "Connect local Ollama models to screenpipe",+ "lmstudio": "Connect LM Studio models to screenpipe",+ "msty": "Connect Msty models to screenpipe",+ "obsidian": "Sync screen memory to your Obsidian vault",+ "notion": "Search Notion pages with your screen context",+ "linear": "Search Linear issues from your screen context",+ "perplexity": "Search the web with Perplexity AI",+ "krisp": "Search Krisp meeting transcripts and notes",+ "plaud": "Search Plaud recordings and transcripts",+ "excalidraw": "Search and edit your Excalidraw+ whiteboards",+ "custom-mcp": "Connect any MCP-compatible server",+ "skills": "Import Claude Code skills for AI automations",+};++// High-activation defaults fill the suggested row when there are not enough+// detected or already-connected apps on the device.+export const FEATURED_CONNECTION_IDS = [+ "custom-mcp",+ "claude",+ "cursor",+ "codex",+ "claude-code",+ "chatgpt",+ "slack",+ "obsidian",+ "notion",+];++export const DEVICE_CONNECTION_ORDER = [+ "custom-mcp",+ "claude",+ "cursor",+ "codex",+ "claude-code",+ "chatgpt",+ "browser-url",+ "input-monitoring",+ "obsidian",+ "notion",+ "linear",+ "slack",+ "gmail",+ "apple-calendar",+ "google-calendar",+ "google-docs",+ "google-sheets",+ "warp",+ "ollama",+ "lmstudio",+ "msty",+ "krisp",+ "whatsapp",+];++export function normalizeConnectionCategory(category: string | null | undefined): string {+ const value = (category || "Other").trim();+ if (!value) return "Other";+ return value+ .split(/[-_\s]+/)+ .filter(Boolean)+ // Preserve all-caps acronyms (AI, CRM) instead of mangling them to "Ai"/"Crm".+ .map((part) =>+ /^[A-Z0-9]{2,}$/.test(part)+ ? part+ : part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()+ )+ .join(" ");+}++function connectionPriority(tile: ConnectionSuggestionTile): number {+ if (tile.connected) return 0;+ if (tile.detected) return 1;+ if (FEATURED_CONNECTION_IDS.includes(tile.id)) return 2;+ return 3;+}++function connectionOrder(tile: ConnectionSuggestionTile): number {+ const index = DEVICE_CONNECTION_ORDER.indexOf(tile.id);+ return index === -1 ? DEVICE_CONNECTION_ORDER.length : index;+}++export function compareConnectionTiles(+ a: ConnectionSuggestionTile,+ b: ConnectionSuggestionTile,+): number {+ const priority = connectionPriority(a) - connectionPriority(b);+ if (priority !== 0) return priority;+ const order = connectionOrder(a) - connectionOrder(b);+ if (order !== 0) return order;+ return a.name.localeCompare(b.name);+}++export function isSuggestedForThisDevice(tile: ConnectionSuggestionTile): boolean {+ return tile.connected || !!tile.detected || FEATURED_CONNECTION_IDS.includes(tile.id);+}++export function getSuggestedConnectionsForDevice<T extends ConnectionSuggestionTile>(+ tiles: T[],+ limit = 8,+): T[] {+ return [...tiles]+ .filter(isSuggestedForThisDevice)+ .sort(compareConnectionTiles)+ .slice(0, limit);+}diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-hardcoded-tiles.ts b/apps/screenpipe-app-tauri/lib/hooks/use-hardcoded-tiles.tsindex ea9ca0dc8..5e82994ad 100644--- a/apps/screenpipe-app-tauri/lib/hooks/use-hardcoded-tiles.ts+++ b/apps/screenpipe-app-tauri/lib/hooks/use-hardcoded-tiles.ts@@ -8,12 +8,14 @@ import { join, homeDir } from "@tauri-apps/api/path";import { platform } from "@tauri-apps/plugin-os";import { commands } from "@/lib/utils/tauri";import { getStore } from "@/lib/hooks/use-settings";+import { localFetch } from "@/lib/api";export interface HardcodedTile {id: string;name: string;icon: string;connected: boolean;+ detected?: boolean;}export async function getClaudeConfigPath(): Promise<string | null> {@@ -81,6 +83,9 @@ export function useHardcodedTiles(): HardcodedTile[] {const [cursorInstalled, setCursorInstalled] = useState(false);const [codexInstalled, setCodexInstalled] = useState(false);const [chatgptConnected, setChatgptConnected] = useState(false);+ const [customMcpConnected, setCustomMcpConnected] = useState(false);+ const [customMcpDetected, setCustomMcpDetected] = useState(false);+ const [inputMonitoringGranted, setInputMonitoringGranted] = useState(false);const [calendarConnected, setCalendarConnected] = useState(false);useEffect(() => {@@ -95,6 +100,29 @@ export function useHardcodedTiles(): HardcodedTile[] {.then(res => setChatgptConnected(res.status === "ok" && res.data.logged_in)).catch(() => {});+ localFetch("/mcp-servers")+ .then(async (res) => {+ if (!res.ok) {+ setCustomMcpConnected(false);+ setCustomMcpDetected(false);+ return;+ }+ const body = await res.json();+ const list = (body?.data ?? []) as { enabled: boolean }[];+ setCustomMcpDetected(list.length > 0);+ setCustomMcpConnected(list.some((server) => server.enabled));+ })+ .catch(() => {+ setCustomMcpConnected(false);+ setCustomMcpDetected(false);+ });++ if (typeof window !== "undefined" && platform() === "macos") {+ commands.checkInputMonitoringPermissionCmd()+ .then(r => setInputMonitoringGranted(r === "granted"))+ .catch(() => setInputMonitoringGranted(false));+ }+getStore().then(store => store.get<boolean>("calendarUserDisconnected")).then(val => setCalendarConnected(!(val ?? false)))@@ -108,11 +136,13 @@ export function useHardcodedTiles(): HardcodedTile[] {{ id: "claude-code", name: "Claude Code", icon: "claude-code", connected: false },{ id: "warp", name: "Warp", icon: "warp", connected: false },{ id: "chatgpt", name: "ChatGPT", icon: "chatgpt", connected: chatgptConnected },+ { id: "custom-mcp", name: "Custom MCP", icon: "custom-mcp", connected: customMcpConnected, detected: customMcpDetected },...(os === "macos" ? [{ id: "browser-url", name: "Browser URL Capture", icon: "browser-url", connected: false },{ id: "voice-memos", name: "Voice Memos", icon: "voice-memos", connected: false },] as HardcodedTile[] : []),...(os === "macos" ? [{ id: "apple-intelligence", name: "Apple Intelligence", icon: "apple-intelligence", connected: false } as HardcodedTile] : []),+ ...(os === "macos" ? [{ id: "input-monitoring", name: "Input Monitoring", icon: "input-monitoring", connected: inputMonitoringGranted } as HardcodedTile] : []),...(os === "macos" ? [{ id: "apple-calendar", name: "Apple Calendar", icon: "apple-calendar", connected: calendarConnected } as HardcodedTile] : []),];}