Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): loopback MCP self-calls behind auth proxy (PRD-8062) (#3572)
omar-inkeep8925162bcb46
diff --git a/.changeset/mcp-write-loopback-behind-proxy.md b/.changeset/mcp-write-loopback-behind-proxy.mdnew file mode 100644index 000000000..2ddab2632--- /dev/null+++ b/.changeset/mcp-write-loopback-behind-proxy.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Fixed MCP write tools (`write`, `edit`, `move`, `delete`, and related) returning `HTTP 401` when the server runs behind an authenticating reverse proxy with `--external-url`/`OK_EXTERNAL_URL` set to the public origin. The tools' internal server-to-self API calls were being issued against the public, auth-gated URL, so they hairpinned out through the proxy and hit the login gate instead of the local server. These self-calls now always target loopback, so a single `OK_EXTERNAL_URL` pointing at your public origin gives working remote UI, reads, and writes at once. User-facing preview and share links are unaffected — they were never derived from this URL.diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.tsindex e0363d8cd..4d3e1edd8 100644--- a/packages/cli/src/commands/start.ts+++ b/packages/cli/src/commands/start.ts@@ -348,8 +348,8 @@ export function parseExternalUrlFlag(value: string): string {* precedence bookkeeping, not conflict resolution). Extracted so a test pins* that the deprecated flag keeps setting `server.externalUrl` through its* removal window: dropping the fallback would silently strand `--public-url`- * operators on `externalUrl: undefined` (no CORS/Host admission, no issued- * URLs) with no parse error to signal it.+ * operators on `externalUrl: undefined` (no CORS/Host admission) with no parse+ * error to signal it.*/export function resolveFlagExternalUrl(opts: Pick<StartCommandOptions, 'externalUrl' | 'publicUrl'>,@@ -567,7 +567,7 @@ interface BootStartServerOptions {idleThresholdMs?: number | null;/*** The fully-layered `server.*` resolution (flags > env > project-local >- * project > user), threaded to `bootServer` so issued URLs and the+ * project > user), threaded to `bootServer` so Host/Origin admission and the* exposure interlock consume the same values the CLI resolved. Omitted by* legacy callers — `bootServer` then resolves files-only from its config.*/@@ -1733,7 +1733,7 @@ export function startCommand(getConfig: () => Config): Command {).option('--external-url <url>',- 'Canonical external origin clients dial (sets server.externalUrl for this run) — its host joins the Host/Origin allowlists and issued URLs. External exposure additionally requires consent (OK_ALLOW_EXTERNAL=1 or server.allowExternal).',+ 'Canonical external origin clients dial (sets server.externalUrl for this run) — its host joins the Host/Origin allowlists (CORS + external-Host admission). External exposure additionally requires consent (OK_ALLOW_EXTERNAL=1 or server.allowExternal).',parseExternalUrlFlag,).option(diff --git a/packages/core/src/config/resolve-server-config.ts b/packages/core/src/config/resolve-server-config.tsindex 100922ea3..96438565d 100644--- a/packages/core/src/config/resolve-server-config.ts+++ b/packages/core/src/config/resolve-server-config.ts@@ -50,7 +50,7 @@ export interface ServerRuntimeConfig {* Load-bearing distinction: a `remote.url` left in config does NOT arm* anything by itself (`ok start --remote` is the explicit opt-in), so* consumers that treat a declared external origin as an exposure signal —- * the consent interlock, issued-URL preference — must key off `'server'`+ * the consent interlock, Host/Origin admission — must key off `'server'`* only and leave the alias to the legacy remote flow.*/externalUrlSource: 'server' | 'remote-alias' | undefined;diff --git a/packages/core/src/config/schema.ts b/packages/core/src/config/schema.tsindex 2434f7940..122dcc77c 100644--- a/packages/core/src/config/schema.ts+++ b/packages/core/src/config/schema.ts@@ -1377,7 +1377,7 @@ export const ConfigSchema = z.looseObject({reload: 'boot',defaultScope: 'project',description:- 'Canonical external origin the server is reached at, e.g. https://kb.example.com — drives issued URLs and CORS. Unset by default: a loopback server derives http://localhost:<port>. Setting it declares external exposure, which additionally requires the server.allowExternal consent interlock. Supersedes server.publicUrl (its former name) and remote.url — both still read while this key is absent. Read at server start; changing it requires a restart.',+ 'Canonical external origin the server is reached at, e.g. https://kb.example.com — its host joins the Host/Origin allowlists (external-Host + CORS admission). Unset by default: the server admits only loopback Hosts. Setting it declares external exposure, which additionally requires the server.allowExternal consent interlock. Supersedes server.publicUrl (its former name) and remote.url — both still read while this key is absent. Read at server start; changing it requires a restart.',}).optional(),// The former name of `server.externalUrl`, shipped in stable 0.51.x —diff --git a/packages/server/src/boot.test.ts b/packages/server/src/boot.test.tsindex 8d5a86ccd..e55ba0284 100644--- a/packages/server/src/boot.test.ts+++ b/packages/server/src/boot.test.ts@@ -736,6 +736,117 @@ describe('bootServer — reactShellDistDir end-to-end HTTP shape', () => {});});+describeEvenOnCI('bootServer — MCP internal RPC stays loopback under a public externalUrl', () => {+ // Regression: behind an authenticating reverse proxy, `ok start+ // --external-url https://<public>` set the MCP tools' `serverUrl` to the+ // public origin, so the document-write self-call (`POST /api/agent-write-md`)+ // hairpinned out through the edge and hit the auth gate — HTTP 401 with an+ // HTML login body. `getServerUrl` must stay loopback regardless of+ // `externalUrl`; the write must land against the local listener.+ //+ // We can't stand up a real proxy in-process, so we point `externalUrl` at an+ // unreachable `.invalid` host (RFC 6761 — guaranteed non-resolvable). With+ // the old wiring the self-call dials that host and fails ("Server+ // unreachable"); with the fix it dials loopback and succeeds. Same fault, one+ // hop earlier — a deterministic proxy-free proxy.+ const MCP_PROTOCOL_VERSION = '2025-06-18';+ const UNREACHABLE_EXTERNAL_URL = 'https://prd-8062-unreachable.invalid';++ async function mcpFetch(port: number, headers: Record<string, string>, body: unknown) {+ return fetch(`http://127.0.0.1:${port}/mcp`, {+ method: 'POST',+ headers: {+ accept: 'application/json, text/event-stream',+ 'content-type': 'application/json',+ ...headers,+ },+ body: JSON.stringify(body),+ });+ }++ test('MCP write succeeds when externalUrl is a public, non-loopback origin', async () => {+ const projectDir = mkdtempSync(resolve(tmpDir, 'mcp-loopback-'));+ seedOkScaffold(projectDir);++ const config = ConfigSchema.parse({+ server: { externalUrl: UNREACHABLE_EXTERNAL_URL, allowExternal: true },+ });+ // Sanity-pin the setup: the runtime really does carry the public origin, so+ // a passing write proves the self-call ignored it (not that it was absent).+ expect(resolveServerRuntimeConfig(config).externalUrl).toBe(UNREACHABLE_EXTERNAL_URL);++ const booted = await bootServer({+ host: '127.0.0.1',+ config,+ projectDir,+ contentDir: projectDir,+ port: 0,+ quiet: true,+ gitEnabled: false,+ idleShutdownMs: null,+ });+ try {+ await booted.ready;+ const port = booted.port;++ // Handshake over loopback (always ingress-admitted).+ const init = await mcpFetch(+ port,+ {},+ {+ jsonrpc: '2.0',+ id: 1,+ method: 'initialize',+ params: {+ protocolVersion: MCP_PROTOCOL_VERSION,+ capabilities: {},+ clientInfo: { name: 'prd-8062-test', version: '0.0.0' },+ },+ },+ );+ expect(init.status).toBe(200);+ const sessionId = init.headers.get('mcp-session-id');+ expect(sessionId).toBeTruthy();+ const sessionHeaders = {+ 'mcp-session-id': sessionId as string,+ 'mcp-protocol-version': MCP_PROTOCOL_VERSION,+ };++ const initialized = await mcpFetch(port, sessionHeaders, {+ jsonrpc: '2.0',+ method: 'notifications/initialized',+ });+ expect(initialized.status).toBe(202);++ const write = await mcpFetch(port, sessionHeaders, {+ jsonrpc: '2.0',+ id: 2,+ method: 'tools/call',+ params: {+ name: 'write',+ arguments: {+ document: { path: 'prd-8062-note', content: '# PRD-8062\n\nLoopback write.\n' },+ cwd: projectDir,+ },+ },+ });+ expect(write.status).toBe(200);+ const body = (await write.json()) as {+ result?: { isError?: boolean; content?: Array<{ text?: string }> };+ error?: unknown;+ };+ expect(body.error).toBeUndefined();+ // The load-bearing assertion: the write self-call landed on loopback, not+ // the public externalUrl. A regressed wiring surfaces here as isError with+ // a "Server unreachable" / 401 body.+ const text = body.result?.content?.map((c) => c.text ?? '').join('') ?? '';+ expect(body.result?.isError ?? false, `write returned an error: ${text}`).toBe(false);+ } finally {+ await booted.destroy();+ }+ });+});+describe('bootServer — ok.boot OTel span attributes', () => {let exporter: InMemorySpanExporter | null = null;let provider: BasicTracerProvider | null = null;diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.tsindex e1088da7d..68b3570ef 100644--- a/packages/server/src/boot.ts+++ b/packages/server/src/boot.ts@@ -249,7 +249,7 @@ export interface BootServerOptions* defaults). The CLI passes the fully-layered resolution (flags > env >* project-local > project > user); callers that omit it get a files-only* resolution from `opts.config`. One resolution feeds every consumer —- * issued URLs, and the exposure interlock — so the CLI and a direct+ * Host/Origin admission, and the exposure interlock — so the CLI and a direct* embedder can never disagree about what the server thinks its shape is.*/serverRuntime?: ServerRuntimeConfig;@@ -831,17 +831,28 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {return host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;})();let boundPort = opts.port ?? 0;- // Issued URLs (MCP serverUrl → preview_url, ACP thread bootstrap) name the- // declared public origin when one is configured via the successor key —- // that is the name clients actually type, and a loopback URL handed to a- // remote agent is unreachable. The remote-alias source (a dormant- // `remote.url` left in config with no `--remote` flag) never rewrites- // issued URLs, and the server.lock URL stays loopback below: the lock is- // same-machine discovery by contract.- const issuedBaseUrl = (): string =>- serverRuntime.externalUrlSource === 'server' && serverRuntime.externalUrl !== undefined- ? serverRuntime.externalUrl.replace(/\/+$/, '')- : `http://${mcpHost}:${boundPort}`;+ // Internal service-to-self base URL — the `serverUrl` the MCP tools and the+ // ACP hosted-agent connect-back dial. Both consumers run on THIS machine:+ // the MCP tools' enrichment reads and the document write/persist self-call+ // (`content/enrichment.ts`, `mcp/tools/write.ts` → `POST /api/agent-write-md`),+ // and the ACP-spawned agent subprocess dialing `${url}/mcp`. So the base MUST+ // target this server's OWN bound listener (`mcpHost` is whatever address the+ // server bound — loopback in the default bind). Behind an authenticating reverse proxy with a public+ // `server.externalUrl`, routing these self-calls at the external origin+ // hairpins them out through the edge and hits the auth gate, which answers+ // `HTTP 401` with an HTML login body — the exact failure that broke MCP+ // write/edit/move/delete (reads survive only because their enrichment+ // self-calls are `.catch(() => null)`).+ //+ // This is NOT where user-facing "issued" links come from, so keeping it+ // loopback does not regress preview/share URLs: `preview_url` and the+ // per-response `previewUrl` route resolve their browser base from+ // `server.lock` / `ui.lock` (this same listener origin by the one-URL+ // contract below, never from this value), and `share_link` builds its public+ // `https://openknowledge.ai/d/...` URL from the git remote. `externalUrl`+ // still governs ingress admission (CORS/Host) via `ingressPolicy`; it just+ // must never feed an internal self-call.+ const internalBaseUrl = (): string => `http://${mcpHost}:${boundPort}`;// No-project ephemeral single-file mode mounts NO MCP endpoint:// there are no agent capabilities. `mountMcpAndApi` leaves `/mcp`// unmounted when `mcpHttpHandler` is undefined, so an undefined handler is@@ -852,7 +863,7 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {contentDir: opts.contentDir,projectDir: opts.projectDir ?? opts.contentDir,config: opts.config,- getServerUrl: () => issuedBaseUrl(),+ getServerUrl: () => internalBaseUrl(),localApi: serverInstance.localApi,log,});@@ -1005,7 +1016,12 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {isIgnoredPath: (rel) => serverInstance.contentFilter.isPathIgnored(rel),getLoadedDocText: (docName) =>hocuspocus.documents.get(docName)?.getText('source').toString() ?? null,- getServerUrl: () => issuedBaseUrl(),+ // MUST be `internalBaseUrl()` (same contract as the MCP-tools site above):+ // the ACP agent is a subprocess on THIS machine dialing `${url}/mcp`, so a+ // public `externalUrl` here would hairpin it through the auth proxy. Unlike+ // the MCP-tools 401, that failure is SILENT (the agent just loses OK tools),+ // so do not repoint this at `externalUrl`.+ getServerUrl: () => internalBaseUrl(),getMcpStdioCommand: () => buildOkMcpStdioCommand(opts.localOpCliArgs, boundPort),probeHarnessManagedMcpEntry: opts.probeHarnessManagedMcpEntry,log,@@ -1215,9 +1231,11 @@ async function bootServerInner(opts: BootServerOptions): Promise<BootedServer> {const addr = httpServer.address();const realPort = typeof addr === 'object' && addr !== null ? addr.port : (opts.port ?? 0);boundPort = realPort;- // One-URL contract: the advertised origin is the same one `getServerUrl`- // hands MCP/ACP consumers — every surface of this listener lives there.- const boundBaseUrl = `http://${mcpHost}:${realPort}`;+ // One-URL contract: the advertised origin is the same base `getServerUrl`+ // hands MCP/ACP consumers. `boundPort` is now `realPort`, so `internalBaseUrl()`+ // yields exactly this listener's origin — reusing it keeps the lock URL and+ // the self-call base structurally identical rather than two literals in sync.+ const boundBaseUrl = internalBaseUrl();updateServerLockPort(lockDir, realPort, boundBaseUrl);if (ownsUiLock) {// Flip the sentinel port=0 to the bound port so preview-URL consumers see