YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(connections): complete Hermes MCP setup alongside existing servers (#7454)
Ezra Ellette241d26d8cda8
diff --git a/apps/screenpipe-app-tauri/bun.lock b/apps/screenpipe-app-tauri/bun.lockindex 161f46e04..f7a7a4e6f 100644--- a/apps/screenpipe-app-tauri/bun.lock+++ b/apps/screenpipe-app-tauri/bun.lock@@ -82,6 +82,7 @@"unified": "^11.0.0","unpdf": "^1.6.2","xlsx": "^0.18.5",+ "yaml": "^2.8.1","zod": "^3","zustand": "^5.0.11",},diff --git a/apps/screenpipe-app-tauri/lib/__tests__/ai-tools-mcp.test.ts b/apps/screenpipe-app-tauri/lib/__tests__/ai-tools-mcp.test.tsindex 7a1d53cd7..b1c627704 100644--- a/apps/screenpipe-app-tauri/lib/__tests__/ai-tools-mcp.test.ts+++ b/apps/screenpipe-app-tauri/lib/__tests__/ai-tools-mcp.test.ts@@ -8,6 +8,7 @@// fails, and disconnect stays idempotent.import { beforeEach, describe, expect, it, vi } from "vitest";+import { parse as parseYaml } from "yaml";const fsMock = vi.hoisted(() => ({files: new Map<string, string>(),@@ -113,6 +114,7 @@ import {installCursorMcp,uninstallCursorMcp,installHermesMcp,+ isHermesMcpInstalled,uninstallHermesMcp,installRunnerMcp,uninstallRunnerMcp,@@ -313,11 +315,51 @@ describe("hermes yaml handling", () => {expect(after).not.toContain("SCREENPIPE_LOCAL_API_KEY");});- it("fails visibly on a hand-authored mcp_servers block", async () => {- fsMock.files.set(HERMES, "mcp_servers:\n other:\n command: x\n");+ it("connects and disconnects alongside existing servers, preserving comments", async () => {+ const seeded = "# my model\nmodel: x\nmcp_servers:\n # keep my server\n other:\n command: x\nnotifications: true\n";+ fsMock.files.set(HERMES, seeded);+ await expect(isToolConfigHealthy("hermes")).resolves.toBe(true);+ await installHermesMcp();+ await expect(isHermesMcpInstalled()).resolves.toBe(true);+ const connected = fsMock.files.get(HERMES)!;+ expect(connected).toContain("# my model");+ expect(connected).toContain("# keep my server");+ expect(parseYaml(connected).mcp_servers.other).toEqual({ command: "x" });+ expect(parseYaml(connected).mcp_servers.screenpipe.env.SCREENPIPE_LOCAL_API_KEY).toBe("sp-test");+ await uninstallHermesMcp();+ expect(parseYaml(fsMock.files.get(HERMES)!)).toEqual(parseYaml(seeded));+ await expect(isHermesMcpInstalled()).resolves.toBe(false);+ });++ it("refreshes stale credentials without duplicating a mixed-case Screenpipe entry", async () => {+ fsMock.files.set(HERMES, "mcp_servers:\n Screenpipe:\n command: old\n env: {SCREENPIPE_LOCAL_API_KEY: stale}\n other: {command: keep}\n");+ await installHermesMcp();+ await installHermesMcp();+ const servers = parseYaml(fsMock.files.get(HERMES)!).mcp_servers;+ expect(Object.keys(servers).sort()).toEqual(["other", "screenpipe"]);+ expect(servers.screenpipe.env.SCREENPIPE_LOCAL_API_KEY).toBe("sp-test");+ });- await expect(installHermesMcp()).rejects.toThrow(/add the screenpipe server manually/);- expect(fsMock.files.get(HERMES)).toBe("mcp_servers:\n other:\n command: x\n");+ it("does not mistake mentions in prompts or other settings for an installed server", async () => {+ fsMock.files.set(HERMES, "prompt: |\n use screenpipe-mcp\nskills:\n screenpipe: true\n");+ await expect(isHermesMcpInstalled()).resolves.toBe(false);+ await installHermesMcp();+ await expect(isHermesMcpInstalled()).resolves.toBe(true);+ });++ it("supports an inline mapping while preserving unrelated YAML", async () => {+ fsMock.files.set(HERMES, "mcp_servers: {other: {command: keep}}\nmodel: x\n");+ await installHermesMcp();+ await uninstallHermesMcp();+ expect(parseYaml(fsMock.files.get(HERMES)!)).toEqual({ model: "x", mcp_servers: {other: {command: "keep"}} });+ });++ it.each(["mcp_servers: [broken", "mcp_servers: nope\n", "mcp_servers: {}\nmcp_servers: {}\n", "- not-a-config\n"])("refuses malformed YAML unchanged: %s", async (seeded) => {+ fsMock.files.set(HERMES, seeded);+ await expect(installHermesMcp()).rejects.toThrow();+ await expect(uninstallHermesMcp()).rejects.toThrow();+ await expect(isToolConfigHealthy("hermes")).resolves.toBe(false);+ expect(fsMock.files.get(HERMES)).toBe(seeded);});});diff --git a/apps/screenpipe-app-tauri/lib/ai-tools-mcp.ts b/apps/screenpipe-app-tauri/lib/ai-tools-mcp.tsindex 2c452e73d..286cf7fa1 100644--- a/apps/screenpipe-app-tauri/lib/ai-tools-mcp.ts+++ b/apps/screenpipe-app-tauri/lib/ai-tools-mcp.ts@@ -34,6 +34,7 @@ import { isGrokBotDetected, grokBotConnection } from "@/lib/grokbot-connection";import { parse, modify, applyEdits, type ParseError } from "jsonc-parser";import { parse as parseToml } from "smol-toml";+import { isMap, isScalar, parseDocument } from "yaml";type McpCommand = { command: string; args: string[]; env?: Record<string, string> };@@ -565,67 +566,48 @@ export async function uninstallOpenclawMcp(): Promise<void> {}// ─── Hermes ──────────────────────────────────────────────────────────────────-// YAML config at ~/.hermes/config.yaml. No YAML parser in the frontend, so we-// only ever add or remove the exact block we write ourselves; anything-// hand-authored fails loudly instead of getting string-sliced.-+// Edit the YAML document tree so comments and unrelated servers survive.+// All writes still use the shared backup, symlink resolution and atomic rename.export async function getHermesConfigPath(): Promise<string> {- const home = await homeDir();- return join(home, ".hermes", "config.yaml");+ return join(await homeDir(), ".hermes", "config.yaml");}-// Hermes ships a commented-out `# mcp_servers:` example block in its default-// config.yaml — substring checks match it and wrongly report a hand-authored-// block. Only uncommented lines count.-const HERMES_MCP_BLOCK = /^mcp_servers:\s*$/m;+function parseHermesConfig(text: string, path: string) {+ const doc = parseDocument(text);+ if (doc.errors.length || (doc.contents !== null && !isMap(doc.contents))) {+ throw new Error(`${path} is not valid YAML — screenpipe won't overwrite it`);+ }+ const servers = doc.get("mcp_servers", true);+ if (servers !== undefined && !isMap(servers)) {+ throw new Error(`${path} has a custom mcp_servers block — expected a mapping`);+ }+ return doc;+}-function hermesHasScreenpipe(content: string): boolean {- return content.split("\n").some(- (l) =>- !l.trimStart().startsWith("#") &&- (/^\s+screenpipe:/i.test(l) || l.toLowerCase().includes("screenpipe-mcp"))+function hermesServerKeys(doc: ReturnType<typeof parseHermesConfig>): string[] {+ const servers = doc.get("mcp_servers", true);+ if (!isMap(servers)) return [];+ return servers.items.flatMap(({ key }) =>+ isScalar(key) && typeof key.value === "string" && key.value.toLowerCase() === "screenpipe"+ ? [key.value] : []);}export async function isHermesMcpInstalled(): Promise<boolean> {try {- return hermesHasScreenpipe(await readTextFile(await getHermesConfigPath()));+ const path = await getHermesConfigPath();+ const doc = parseHermesConfig((await readConfigText(path)) ?? "", path);+ return hermesServerKeys(doc).some(key => isMap(doc.getIn(["mcp_servers", key], true)));} catch { return false; }}export async function installHermesMcp(): Promise<McpCommand> {const configPath = await getHermesConfigPath();+ const doc = parseHermesConfig((await readConfigText(configPath)) ?? "", configPath);const mcp = await buildMcpConfig({ client: "hermes" });- const { command, args, env } = mcp;- const existing = (await readConfigText(configPath)) ?? "";-- if (hermesHasScreenpipe(existing)) {- return mcp; // already wired — leave hand-edited YAML alone- }-- const envBlock =- env && Object.keys(env).length > 0- ? `\n env:\n${Object.entries(env)- .map(([k, v]) => ` ${k}: ${JSON.stringify(v)}`)- .join("\n")}`- : "";- const server = ` screenpipe:\n command: ${JSON.stringify(command)}\n args:\n${args- .map((a) => ` - ${JSON.stringify(a)}`)- .join("\n")}${envBlock}\n`;-- if (HERMES_MCP_BLOCK.test(existing)) {- // A real (uncommented) hand-authored block we can't safely string-merge- // into. Fail loudly so the per-tool status shows failed instead of a- // silent fake success.- throw new Error(- "~/.hermes/config.yaml already has an mcp_servers block — add the screenpipe server manually"- );- }-- let out = existing;- if (out && !out.endsWith("\n")) out += "\n";- out += `mcp_servers:\n${server}`;- await replaceConfig(configPath, out);+ for (const key of hermesServerKeys(doc)) doc.deleteIn(["mcp_servers", key]);+ doc.setIn(["mcp_servers", "screenpipe"], mcp);+ await replaceConfig(configPath, doc.toString());return mcp;}@@ -633,35 +615,13 @@ export async function uninstallHermesMcp(): Promise<void> {const configPath = await getHermesConfigPath();const existing = await readConfigText(configPath);if (existing === null) return;-- // Strip exactly the block installHermesMcp writes: the `mcp_servers:` line- // plus its indented children — but only when screenpipe is its sole child.- // No real (uncommented) block means nothing we wrote — no-op.- const lines = existing.split("\n");- const start = lines.findIndex((l) => /^mcp_servers:\s*$/.test(l));- if (start === -1) return;- let end = start + 1;- const topLevelChildren: string[] = [];- while (end < lines.length && (/^\s+\S/.test(lines[end]) || lines[end].trim() === "")) {- const m = lines[end].match(/^ (\S[^:]*):/);- if (m) topLevelChildren.push(m[1]);- end++;- }- const onlyScreenpipe =- topLevelChildren.length === 1 &&- topLevelChildren[0].toLowerCase() === "screenpipe";- const blockText = lines.slice(start, end).join("\n");- if (!onlyScreenpipe || !blockText.toLowerCase().includes("screenpipe-mcp")) {- throw new Error(- "~/.hermes/config.yaml has a customized mcp_servers block — remove the screenpipe entry manually"- );- }-- const next = [...lines.slice(0, start), ...lines.slice(end)]- .join("\n")- .replace(/\n{3,}/g, "\n\n")- .replace(/^\n+/, "");- await replaceConfig(configPath, next);+ const doc = parseHermesConfig(existing, configPath);+ const keys = hermesServerKeys(doc);+ if (!keys.length) return;+ for (const key of keys) doc.deleteIn(["mcp_servers", key]);+ const servers = doc.get("mcp_servers", true);+ if (isMap(servers) && !servers.items.length) doc.delete("mcp_servers");+ await replaceConfig(configPath, doc.toString());}// VS Code's default user profile uses JSONC and a `servers` map.@@ -1020,7 +980,7 @@ export function friendlyToolError(err: unknown): FriendlyToolError {// Cause only — the fix is inferable from the retry button + open-file// action every surface renders next to this message. No embedded paths:// that's the open-file button's job.- if (detail.includes("not valid JSON")) {+ if (detail.includes("not valid JSON") || detail.includes("not valid YAML")) {return { message: "config file has a syntax error", path, detail };}if (detail.includes("could not read")) {@@ -1087,9 +1047,9 @@ export async function isToolConfigHealthy(id: ConnectAllToolId): Promise<booleanawait readConfigText(await getCodexConfigPath());return true;case "hermes": {- const text = (await readConfigText(await getHermesConfigPath())) ?? "";- // Our one refusal: a hand-authored mcp_servers block without screenpipe.- return hermesHasScreenpipe(text) || !HERMES_MCP_BLOCK.test(text);+ const path = await getHermesConfigPath();+ parseHermesConfig((await readConfigText(path)) ?? "", path);+ return true;}}} catch {diff --git a/apps/screenpipe-app-tauri/package.json b/apps/screenpipe-app-tauri/package.jsonindex ab5151d80..7207966e1 100644--- a/apps/screenpipe-app-tauri/package.json+++ b/apps/screenpipe-app-tauri/package.json@@ -166,6 +166,7 @@"unified": "^11.0.0","unpdf": "^1.6.2","xlsx": "^0.18.5",+ "yaml": "^2.8.1","zod": "^3","zustand": "^5.0.11"},diff --git a/docs/mintlify/docs-mintlify-mig-tmp/docs.json b/docs/mintlify/docs-mintlify-mig-tmp/docs.jsonindex 1b6ae1e32..895dcb965 100644--- a/docs/mintlify/docs-mintlify-mig-tmp/docs.json+++ b/docs/mintlify/docs-mintlify-mig-tmp/docs.json@@ -132,6 +132,7 @@"claude-code","opencode","openclaw",+ "hermes","cline","continue","gemini-cli",diff --git a/docs/mintlify/docs-mintlify-mig-tmp/hermes.mdx b/docs/mintlify/docs-mintlify-mig-tmp/hermes.mdxnew file mode 100644index 000000000..b07108194--- /dev/null+++ b/docs/mintlify/docs-mintlify-mig-tmp/hermes.mdx@@ -0,0 +1,44 @@+---+title: "Hermes - Screenpipe context for your agent"+sidebarTitle: "Hermes"+description: "Connect Hermes to Screenpipe with MCP and skills to recall screen history and meetings."+icon: "brain"+---++[Hermes](https://hermes-agent.nousresearch.com) can query Screenpipe through MCP. Screenpipe's skills teach the agent workflows such as recall, meeting prep, and worklogs.++## Connect on the same computer++1. Start Screenpipe and open Hermes once so its configuration directory exists.+2. In Screenpipe, open **Settings → Connections → AI apps** and connect **Hermes**.+3. Restart Hermes, or run `/reload-mcp` inside Hermes.++Screenpipe adds its MCP server to `~/.hermes/config.yaml` and installs its bundled skills in `~/.hermes/skills/`. Other MCP servers and unrelated settings are preserved. Reconnecting refreshes Screenpipe's launch command and local API credential.++For a CLI installation, use:++```bash+bun x screenpipe@latest agent setup hermes+```++## Verify retrieval++Ask Hermes:++> Use Screenpipe to find what I worked on in the last 10 minutes. Include timestamps and source references. Tell me if the query fails or returns no recordings.++Then try:++> Use Screenpipe to find my latest recorded meeting and summarize its decisions, citing the source.++The connection indicator confirms configuration and skills are installed. A successful query from Hermes verifies that the running agent can access recordings. An empty result does not prove recording is healthy.++## Disconnect++Use **Remove** beside Hermes in Screenpipe's AI apps list, then reload Hermes's MCP connections. Screenpipe removes its entry while preserving other servers and user-edited skills.++## Remote Hermes++When Hermes runs on another computer, localhost refers to that computer. Use a private authenticated connection to the recording computer; see the [remote MCP guide](/mcp-server). Screenpipe's stdio MCP has the full tool set. The HTTP transport currently exposes search and bundled-skill access, so do not expect meeting and activity tools through HTTP yet.++Hermes's API server is a separate, optional connection for sending work **to Hermes**. It is not required for Hermes to read Screenpipe context.diff --git a/docs/pr-assets/hermes-integration/after.png b/docs/pr-assets/hermes-integration/after.pngnew file mode 100644index 000000000..13ad8b528Binary files /dev/null and b/docs/pr-assets/hermes-integration/after.png differdiff --git a/docs/pr-assets/hermes-integration/before.png b/docs/pr-assets/hermes-integration/before.pngnew file mode 100644index 000000000..b5e4b831dBinary files /dev/null and b/docs/pr-assets/hermes-integration/before.png differdiff --git a/docs/pr-assets/hermes-integration/generated-config.yaml b/docs/pr-assets/hermes-integration/generated-config.yamlnew file mode 100644index 000000000..02fae9652--- /dev/null+++ b/docs/pr-assets/hermes-integration/generated-config.yaml@@ -0,0 +1,13 @@+# Personal settings+model: example-model+mcp_servers:+ existing:+ command: existing-server+ screenpipe:+ command: /mock/screenpipe/bun+ args:+ - x+ - screenpipe-mcp@latest+ env:+ SCREENPIPE_MCP_CLIENT: hermes+ SCREENPIPE_LOCAL_API_KEY: sp-fixture-onlydiff --git a/packages/screenpipe-mcp/scripts/hermes-smoke.py b/packages/screenpipe-mcp/scripts/hermes-smoke.pynew file mode 100644index 000000000..2365ff81a--- /dev/null+++ b/packages/screenpipe-mcp/scripts/hermes-smoke.py@@ -0,0 +1,55 @@+# screenpipe — AI that knows everything you've seen, said, or heard+# https://screenpipe.com++"""Run with Hermes's Python: hermes-smoke.py HERMES_REPO GENERATED_CONFIG.+Uses an isolated profile and synthetic authenticated API; no model or recording access.+"""+import os, sys, tempfile, threading, json+from pathlib import Path+from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler+from urllib.parse import urlparse, parse_qs+root = Path(tempfile.mkdtemp(prefix='screenpipe-hermes-smoke-'))+os.environ['HERMES_HOME'] = str(root)+os.environ['SCREENPIPE_LOCAL_API_KEY'] = 'sp-fixture-only'+requests = []+reject = False+class API(BaseHTTPRequestHandler):+ def log_message(self, *args): pass+ def do_GET(self):+ requests.append({'path': self.path, 'authorized': self.headers.get('Authorization') == 'Bearer sp-fixture-only'})+ if reject or not requests[-1]['authorized']:+ self.send_response(401); self.end_headers(); self.wfile.write(b'{"error":"fixture authorization required"}'); return+ self.send_response(200); self.send_header('Content-Type','application/json'); self.end_headers()+ self.wfile.write(json.dumps({'data':[{'type':'OCR','content':{'frame_id':42,'text':'Synthetic Hermes integration acceptance','timestamp':'2026-10-02T12:30:00Z','app_name':'Fixture','window_name':'Acceptance'}}], 'pagination':{'limit':1,'offset':0,'total':1}}).encode())+ def do_POST(self):+ self.send_response(200); self.end_headers(); self.wfile.write(b'{}')+api = ThreadingHTTPServer(('127.0.0.1',0), API)+threading.Thread(target=api.serve_forever,daemon=True).start()+os.environ['SCREENPIPE_API_URL'] = f'http://127.0.0.1:{api.server_port}'+sys.path.insert(0, sys.argv[1])+from tools import mcp_tool+import yaml, shutil+config = yaml.safe_load(Path(sys.argv[2]).read_text())['mcp_servers']+config = {'screenpipe': config['screenpipe']}+# Exercise the generated entry with this checkout's MCP build and a fixture API.+# Only the executable/package location and API address differ from desktop setup.+config['screenpipe']['command'] = shutil.which('bun')+config['screenpipe']['args'] = [str(Path(__file__).resolve().parents[1]/'dist/cli.js')]+config['screenpipe']['env']['SCREENPIPE_API_URL'] = os.environ['SCREENPIPE_API_URL']++try:+ names=mcp_tool.register_mcp_servers(config)+ assert any(n.endswith('search_content') for n in names), names+ from tools.registry import registry+ result=registry.dispatch(next(n for n in names if n.endswith('search_content')),{'start_time':'2026-10-02T12:00:00Z','end_time':'2026-10-02T13:00:00Z','limit':1})+ assert 'Synthetic Hermes integration acceptance' in str(result), result+ assert '2026-10-02T12:30:00Z' in str(result), result+ req=next(r for r in requests if '/search?' in r['path'])+ query=parse_qs(urlparse(req['path']).query)+ assert req['authorized'] and query['start_time']==['2026-10-02T12:00:00Z'] and query['end_time']==['2026-10-02T13:00:00Z']+ reject = True+ denied = registry.dispatch(next(n for n in names if n.endswith('search_content')), {'limit':1})+ assert '401' in str(denied), denied+ print(json.dumps({'passed':True,'tools_discovered':len(names),'authenticated_bounded_search':True,'timestamped_source_returned':True,'auth_failure_reported':True,'data':'synthetic','profile':str(root)}))+finally:+ mcp_tool.shutdown_mcp_servers(); api.shutdown()