Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

ix(chat): show device connection suggestions in composer (#4072)

Ansh Grover2778aae64374
diff --git a/apps/screenpipe-app-tauri/components/settings/connections-section.tsx b/apps/screenpipe-app-tauri/components/settings/connections-section.tsx
index b3a3ec7aa..5407c5da7 100644
--- a/apps/screenpipe-app-tauri/components/settings/connections-section.tsx
+++ b/apps/screenpipe-app-tauri/components/settings/connections-section.tsx
@@ -19,7 +19,14 @@ import { useSettings } from "@/lib/hooks/use-settings";
import { useToast } from "@/components/ui/use-toast";
import { ensureChatGptPreset } from "@/lib/utils/chatgpt-preset";
import { notifyConnectionsUpdated } from "@/lib/connections-events";
-import { CONNECTION_CATEGORY_BY_ID } from "@/lib/constants/connections";
+import {
+ CONNECTION_CATEGORY_BY_ID,
+ CONNECTION_HARDCODED_DESCRIPTIONS,
+ compareConnectionTiles,
+ getSuggestedConnectionsForDevice,
+ normalizeConnectionCategory,
+ type ConnectionSuggestionTile,
+} from "@/lib/constants/connections";
import { Command } from "@tauri-apps/plugin-shell";
import { openUrl } from "@tauri-apps/plugin-opener";
import { message, open as openDialog } from "@tauri-apps/plugin-dialog";
@@ -781,131 +788,12 @@ export function IntegrationIcon({
// Connection tile (compact grid item)
// ---------------------------------------------------------------------------
-interface ConnectionTile {
- id: string;
- name: string;
- icon: string;
- connected: boolean;
- detected?: boolean;
- category?: string;
- description?: string;
-}
-
-const HARDCODED_DESCRIPTIONS: Record<string, string> = {
- "claude": "Search your screen & audio from Claude Desktop via MCP",
- "cursor": "Give Cursor AI access to your screen history via MCP",
- "codex": "Give Codex access to your screen & audio via MCP",
- "claude-code": "Add screen memory to the Claude Code CLI",
- "warp": "Search screen history from Warp terminal via MCP",
- "chatgpt": "Search your screen history from ChatGPT",
- "browser-url": "Capture visited URLs from your browser in real time",
- "voice-memos": "Sync Apple Voice Memos for AI-powered search",
- "apple-intelligence": "Connect Apple Intelligence writing tools",
- "input-monitoring": "Track keyboard & mouse for productivity insights",
- "apple-calendar": "Search Apple Calendar events with AI",
- "google-calendar": "Search Google Calendar events with AI",
- "google-docs": "Read and search your Google Docs",
- "google-sheets": "Read and search your Google Sheets",
- "gmail": "Read and search your Gmail inbox",
- "ics-calendar": "Subscribe to any ICS calendar feed",
- "openclaw": "Browse the web with OpenClaw agents",
- "hermes": "AI-powered messaging assistant",
- "whatsapp": "Search your WhatsApp conversations",
- "anythingllm": "Give AnythingLLM access to your screen",
- "ollama": "Connect local Ollama models to screenpipe",
- "lmstudio": "Connect LM Studio models to screenpipe",
- "msty": "Connect Msty models to screenpipe",
- "obsidian": "Sync screen memory to your Obsidian vault",
- "notion": "Search Notion pages with your screen context",
- "linear": "Search Linear issues from your screen context",
- "perplexity": "Search the web with Perplexity AI",
- "krisp": "Search Krisp meeting transcripts and notes",
- "plaud": "Search Plaud recordings and transcripts",
- "excalidraw": "Search and edit your Excalidraw+ whiteboards",
- "custom-mcp": "Connect any MCP-compatible server",
- "skills": "Import Claude Code skills for AI automations",
-};
+type ConnectionTile = ConnectionSuggestionTile & { icon: string };
type ConnectionSort = "suggested" | "alphabetical";
const ALL_CONNECTION_CATEGORIES = "All";
-// High-activation defaults fill the suggested row when there are not enough
-// detected or already-connected apps on the device.
-const FEATURED_CONNECTION_IDS = [
- "custom-mcp",
- "claude",
- "cursor",
- "codex",
- "claude-code",
- "chatgpt",
- "slack",
- "obsidian",
- "notion",
-];
-
-
-const DEVICE_CONNECTION_ORDER = [
- "custom-mcp",
- "claude",
- "cursor",
- "codex",
- "claude-code",
- "chatgpt",
- "browser-url",
- "input-monitoring",
- "obsidian",
- "notion",
- "linear",
- "slack",
- "gmail",
- "apple-calendar",
- "google-calendar",
- "google-docs",
- "google-sheets",
- "warp",
- "ollama",
- "lmstudio",
- "msty",
- "krisp",
- "whatsapp",
-];
-
-function normalizeConnectionCategory(category: string | null | undefined): string {
- const value = (category || "Other").trim();
- if (!value) return "Other";
- return value
- .split(/[-_\s]+/)
- .filter(Boolean)
- // Preserve all-caps acronyms (AI, CRM) instead of mangling them to "Ai"/"Crm".
- .map((part) =>
- /^[A-Z0-9]{2,}$/.test(part)
- ? part
- : part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()
- )
- .join(" ");
-}
-
-function connectionPriority(tile: ConnectionTile): number {
- if (tile.connected) return 0;
- if (tile.detected) return 1;
- if (FEATURED_CONNECTION_IDS.includes(tile.id)) return 2;
- return 3;
-}
-
-function connectionOrder(tile: ConnectionTile): number {
- const index = DEVICE_CONNECTION_ORDER.indexOf(tile.id);
- return index === -1 ? DEVICE_CONNECTION_ORDER.length : index;
-}
-
-function compareConnectionTiles(a: ConnectionTile, b: ConnectionTile): number {
- const priority = connectionPriority(a) - connectionPriority(b);
- if (priority !== 0) return priority;
- const order = connectionOrder(a) - connectionOrder(b);
- if (order !== 0) return order;
- return a.name.localeCompare(b.name);
-}
-
// Per-connection quickstart prompts shown when "Try in Chat" is clicked.
export const TRY_IN_CHAT_PROMPTS: Record<string, string> = {
@@ -3627,7 +3515,7 @@ export function ConnectionsSection({
...tile,
// Our explicit map overrides the API's category so known tools always land in the right group
category: CONNECTION_CATEGORY_BY_ID[tile.id] ?? tile.category ?? "Other",
- description: tile.description ?? HARDCODED_DESCRIPTIONS[tile.id],
+ description: tile.description ?? CONNECTION_HARDCODED_DESCRIPTIONS[tile.id],
}));
}, [os, claudeInstalled, cursorInstalled, codexInstalled, chatgptConnected, browserUrlConnected, browserUrlDetected, integrations, appleCalendarConnected, googleCalendarConnected, googleDocsConnected, googleSheetsConnected, gmailConnected, customMcpConnected, customMcpServerCount, krispConnected, plaudConnected, excalidrawConnected, inputMonitoringGranted, importedSkillsCount, detectedConnectionIds]);
@@ -3635,14 +3523,7 @@ export function ConnectionsSection({
const suggested = useMemo(() => {
if (!isDefaultView) return [];
- return [...allTiles]
- .filter((tile) => (
- tile.connected ||
- tile.detected ||
- FEATURED_CONNECTION_IDS.includes(tile.id)
- ))
- .sort(compareConnectionTiles)
- .slice(0, 8);
+ return getSuggestedConnectionsForDevice(allTiles, 8);
}, [allTiles, isDefaultView]);
// Flat search results (used when search is active or category is programmatically focused)
diff --git a/apps/screenpipe-app-tauri/components/standalone-chat.tsx b/apps/screenpipe-app-tauri/components/standalone-chat.tsx
index 19f417e1b..dca812fdf 100644
--- a/apps/screenpipe-app-tauri/components/standalone-chat.tsx
+++ b/apps/screenpipe-app-tauri/components/standalone-chat.tsx
@@ -67,6 +67,7 @@ import { buildChipModelContent, buildChipDisplayContent, parseConnectionChip } f
import { getCurrentWindow } from "@tauri-apps/api/window";
import { getCurrentWebview } from "@tauri-apps/api/webview";
import { usePlatform } from "@/lib/hooks/use-platform";
+import { useHardcodedTiles } from "@/lib/hooks/use-hardcoded-tiles";
import { useIsFullscreen } from "@/lib/hooks/use-is-fullscreen";
import { useChatFilePreview } from "@/lib/hooks/use-chat-file-preview";
import { useSqlAutocomplete, useTagAutocomplete } from "@/lib/hooks/use-sql-autocomplete";
@@ -119,6 +120,12 @@ import {
import { usePipes } from "@/lib/hooks/use-pipes";
import { localFetch, getApiBaseUrl } from "@/lib/api";
import { CONNECTIONS_UPDATED_EVENT } from "@/lib/connections-events";
+import {
+ CONNECTION_CATEGORY_BY_ID,
+ CONNECTION_HARDCODED_DESCRIPTIONS,
+ getSuggestedConnectionsForDevice,
+ normalizeConnectionCategory,
+} from "@/lib/constants/connections";
import {
computeChatCitationPlan,
formatSourceCitationsMarkdown,
@@ -2504,6 +2511,7 @@ export function StandaloneChat({
} = {}) {
const { settings, updateSettings, isSettingsLoaded, reloadStore } = useSettings();
const { isMac, isWindows, isLoading: isPlatformLoading } = usePlatform();
+ const hardcodedConnectionTiles = useHardcodedTiles();
// Drop the macOS traffic-light reservation when the window is fullscreen
// (the buttons hide). Only relevant in standalone mode (no parent
// className) — the embedded variant is below the host's chrome anyway.
@@ -2526,6 +2534,30 @@ export function StandaloneChat({
() => buildConnectionSetupSuggestions(allConnectionItems, appItems),
[allConnectionItems, appItems]
);
+ const suggestedConnectionTiles = React.useMemo(() => {
+ const apiById = new Map(allConnectionItems.map((connection) => [connection.id, connection]));
+ const hardcodedIds = new Set(hardcodedConnectionTiles.map((connection) => connection.id));
+ const hardcodedTiles = hardcodedConnectionTiles.map((connection) => {
+ const apiConnection = apiById.get(connection.id);
+ return {
+ ...connection,
+ icon: connection.icon || apiConnection?.icon || connection.id,
+ connected: apiConnection?.connected ?? connection.connected,
+ category: CONNECTION_CATEGORY_BY_ID[connection.id] ?? normalizeConnectionCategory(apiConnection?.category),
+ description: apiConnection?.description ?? CONNECTION_HARDCODED_DESCRIPTIONS[connection.id],
+ };
+ });
+ const apiTiles = allConnectionItems
+ .filter((connection) => !hardcodedIds.has(connection.id) && connection.id !== "owned-default")
+ .map((connection) => ({
+ ...connection,
+ icon: connection.icon || connection.id,
+ category: CONNECTION_CATEGORY_BY_ID[connection.id] ?? normalizeConnectionCategory(connection.category),
+ description: connection.description ?? CONNECTION_HARDCODED_DESCRIPTIONS[connection.id],
+ }));
+
+ return getSuggestedConnectionsForDevice([...hardcodedTiles, ...apiTiles], 8);
+ }, [allConnectionItems, hardcodedConnectionTiles]);
const refreshConnectionState = React.useCallback(async () => {
if (isPlatformLoading) return;
try {
@@ -9582,9 +9614,7 @@ export function StandaloneChat({
Connect your apps to get better answers
</button>
<div className="flex items-center gap-1">
- {connections
- .filter((c) => INTEGRATION_ICON_KEYS.has(c.icon || c.id))
- .slice(0, 8)
+ {suggestedConnectionTiles
.map((c) => (
<button
key={c.id}
diff --git a/apps/screenpipe-app-tauri/lib/constants/connections.ts b/apps/screenpipe-app-tauri/lib/constants/connections.ts
index e3e4455d1..a34dfd07b 100644
--- a/apps/screenpipe-app-tauri/lib/constants/connections.ts
+++ b/apps/screenpipe-app-tauri/lib/constants/connections.ts
@@ -141,3 +141,140 @@ export const CONNECTION_CATEGORY_BY_ID: Record<string, string> = {
"apple-intelligence": "System",
"voice-memos": "System",
};
+
+export interface ConnectionSuggestionTile {
+ id: string;
+ name: string;
+ icon?: string;
+ connected: boolean;
+ detected?: boolean;
+ category?: string;
+ description?: string;
+}
+
+export const CONNECTION_HARDCODED_DESCRIPTIONS: Record<string, string> = {
+ "claude": "Search your screen & audio from Claude Desktop via MCP",
+ "cursor": "Give Cursor AI access to your screen history via MCP",
+ "codex": "Give Codex access to your screen & audio via MCP",
+ "claude-code": "Add screen memory to the Claude Code CLI",
+ "warp": "Search screen history from Warp terminal via MCP",
+ "chatgpt": "Search your screen history from ChatGPT",
+ "browser-url": "Capture visited URLs from your browser in real time",
+ "voice-memos": "Sync Apple Voice Memos for AI-powered search",
+ "apple-intelligence": "Connect Apple Intelligence writing tools",
+ "input-monitoring": "Track keyboard & mouse for productivity insights",
+ "apple-calendar": "Search Apple Calendar events with AI",
+ "google-calendar": "Search Google Calendar events with AI",
+ "google-docs": "Read and search your Google Docs",
+ "google-sheets": "Read and search your Google Sheets",
+ "gmail": "Read and search your Gmail inbox",
+ "ics-calendar": "Subscribe to any ICS calendar feed",
+ "openclaw": "Browse the web with OpenClaw agents",
+ "hermes": "AI-powered messaging assistant",
+ "whatsapp": "Search your WhatsApp conversations",
+ "anythingllm": "Give AnythingLLM access to your screen",
+ "ollama": "Connect local Ollama models to screenpipe",
+ "lmstudio": "Connect LM Studio models to screenpipe",
+ "msty": "Connect Msty models to screenpipe",
+ "obsidian": "Sync screen memory to your Obsidian vault",
+ "notion": "Search Notion pages with your screen context",
+ "linear": "Search Linear issues from your screen context",
+ "perplexity": "Search the web with Perplexity AI",
+ "krisp": "Search Krisp meeting transcripts and notes",
+ "plaud": "Search Plaud recordings and transcripts",
+ "excalidraw": "Search and edit your Excalidraw+ whiteboards",
+ "custom-mcp": "Connect any MCP-compatible server",
+ "skills": "Import Claude Code skills for AI automations",
+};
+
+// High-activation defaults fill the suggested row when there are not enough
+// detected or already-connected apps on the device.
+export const FEATURED_CONNECTION_IDS = [
+ "custom-mcp",
+ "claude",
+ "cursor",
+ "codex",
+ "claude-code",
+ "chatgpt",
+ "slack",
+ "obsidian",
+ "notion",
+];
+
+export const DEVICE_CONNECTION_ORDER = [
+ "custom-mcp",
+ "claude",
+ "cursor",
+ "codex",
+ "claude-code",
+ "chatgpt",
+ "browser-url",
+ "input-monitoring",
+ "obsidian",
+ "notion",
+ "linear",
+ "slack",
+ "gmail",
+ "apple-calendar",
+ "google-calendar",
+ "google-docs",
+ "google-sheets",
+ "warp",
+ "ollama",
+ "lmstudio",
+ "msty",
+ "krisp",
+ "whatsapp",
+];
+
+export function normalizeConnectionCategory(category: string | null | undefined): string {
+ const value = (category || "Other").trim();
+ if (!value) return "Other";
+ return value
+ .split(/[-_\s]+/)
+ .filter(Boolean)
+ // Preserve all-caps acronyms (AI, CRM) instead of mangling them to "Ai"/"Crm".
+ .map((part) =>
+ /^[A-Z0-9]{2,}$/.test(part)
+ ? part
+ : part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()
+ )
+ .join(" ");
+}
+
+function connectionPriority(tile: ConnectionSuggestionTile): number {
+ if (tile.connected) return 0;
+ if (tile.detected) return 1;
+ if (FEATURED_CONNECTION_IDS.includes(tile.id)) return 2;
+ return 3;
+}
+
+function connectionOrder(tile: ConnectionSuggestionTile): number {
+ const index = DEVICE_CONNECTION_ORDER.indexOf(tile.id);
+ return index === -1 ? DEVICE_CONNECTION_ORDER.length : index;
+}
+
+export function compareConnectionTiles(
+ a: ConnectionSuggestionTile,
+ b: ConnectionSuggestionTile,
+): number {
+ const priority = connectionPriority(a) - connectionPriority(b);
+ if (priority !== 0) return priority;
+ const order = connectionOrder(a) - connectionOrder(b);
+ if (order !== 0) return order;
+ return a.name.localeCompare(b.name);
+}
+
+export function isSuggestedForThisDevice(tile: ConnectionSuggestionTile): boolean {
+ return tile.connected || !!tile.detected || FEATURED_CONNECTION_IDS.includes(tile.id);
+}
+
+export function getSuggestedConnectionsForDevice<T extends ConnectionSuggestionTile>(
+ tiles: T[],
+ limit = 8,
+): T[] {
+ return [...tiles]
+ .filter(isSuggestedForThisDevice)
+ .sort(compareConnectionTiles)
+ .slice(0, limit);
+}
diff --git a/apps/screenpipe-app-tauri/lib/hooks/use-hardcoded-tiles.ts b/apps/screenpipe-app-tauri/lib/hooks/use-hardcoded-tiles.ts
index ea9ca0dc8..5e82994ad 100644
--- a/apps/screenpipe-app-tauri/lib/hooks/use-hardcoded-tiles.ts
+++ b/apps/screenpipe-app-tauri/lib/hooks/use-hardcoded-tiles.ts
@@ -8,12 +8,14 @@ import { join, homeDir } from "@tauri-apps/api/path";
import { platform } from "@tauri-apps/plugin-os";
import { commands } from "@/lib/utils/tauri";
import { getStore } from "@/lib/hooks/use-settings";
+import { localFetch } from "@/lib/api";
export interface HardcodedTile {
id: string;
name: string;
icon: string;
connected: boolean;
+ detected?: boolean;
}
export async function getClaudeConfigPath(): Promise<string | null> {
@@ -81,6 +83,9 @@ export function useHardcodedTiles(): HardcodedTile[] {
const [cursorInstalled, setCursorInstalled] = useState(false);
const [codexInstalled, setCodexInstalled] = useState(false);
const [chatgptConnected, setChatgptConnected] = useState(false);
+ const [customMcpConnected, setCustomMcpConnected] = useState(false);
+ const [customMcpDetected, setCustomMcpDetected] = useState(false);
+ const [inputMonitoringGranted, setInputMonitoringGranted] = useState(false);
const [calendarConnected, setCalendarConnected] = useState(false);
useEffect(() => {
@@ -95,6 +100,29 @@ export function useHardcodedTiles(): HardcodedTile[] {
.then(res => setChatgptConnected(res.status === "ok" && res.data.logged_in))
.catch(() => {});
+ localFetch("/mcp-servers")
+ .then(async (res) => {
+ if (!res.ok) {
+ setCustomMcpConnected(false);
+ setCustomMcpDetected(false);
+ return;
+ }
+ const body = await res.json();
+ const list = (body?.data ?? []) as { enabled: boolean }[];
+ setCustomMcpDetected(list.length > 0);
+ setCustomMcpConnected(list.some((server) => server.enabled));
+ })
+ .catch(() => {
+ setCustomMcpConnected(false);
+ setCustomMcpDetected(false);
+ });
+
+ if (typeof window !== "undefined" && platform() === "macos") {
+ commands.checkInputMonitoringPermissionCmd()
+ .then(r => setInputMonitoringGranted(r === "granted"))
+ .catch(() => setInputMonitoringGranted(false));
+ }
+
getStore()
.then(store => store.get<boolean>("calendarUserDisconnected"))
.then(val => setCalendarConnected(!(val ?? false)))
@@ -108,11 +136,13 @@ export function useHardcodedTiles(): HardcodedTile[] {
{ id: "claude-code", name: "Claude Code", icon: "claude-code", connected: false },
{ id: "warp", name: "Warp", icon: "warp", connected: false },
{ id: "chatgpt", name: "ChatGPT", icon: "chatgpt", connected: chatgptConnected },
+ { id: "custom-mcp", name: "Custom MCP", icon: "custom-mcp", connected: customMcpConnected, detected: customMcpDetected },
...(os === "macos" ? [
{ id: "browser-url", name: "Browser URL Capture", icon: "browser-url", connected: false },
{ id: "voice-memos", name: "Voice Memos", icon: "voice-memos", connected: false },
] as HardcodedTile[] : []),
...(os === "macos" ? [{ id: "apple-intelligence", name: "Apple Intelligence", icon: "apple-intelligence", connected: false } as HardcodedTile] : []),
+ ...(os === "macos" ? [{ id: "input-monitoring", name: "Input Monitoring", icon: "input-monitoring", connected: inputMonitoringGranted } as HardcodedTile] : []),
...(os === "macos" ? [{ id: "apple-calendar", name: "Apple Calendar", icon: "apple-calendar", connected: calendarConnected } as HardcodedTile] : []),
];
}