Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Keep clone and init JSON stdout parseable (#7690)
Nick Gomezbc3b44f7c863
diff --git a/.changeset/ok-json-stdout.md b/.changeset/ok-json-stdout.mdnew file mode 100644index 000000000..b4126ccca--- /dev/null+++ b/.changeset/ok-json-stdout.md@@ -0,0 +1,5 @@+---+'@inkeep/open-knowledge': patch+---++Keep `ok clone --json` and `ok init --json` output parseable by sending initialization diagnostics to stderr.diff --git a/knip.config.ts b/knip.config.tsindex e04bf5228..fac2580fe 100644--- a/knip.config.ts+++ b/knip.config.ts@@ -157,11 +157,15 @@ export default {'src/shadow-fixture-launch.test-helper.mjs',],project: 'src/**',- ignoreDependencies: ['@types/shell-quote'],+ ignoreDependencies: [+ '@types/shell-quote',+ ],},'packages/cli': {entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'],- ignoreDependencies: ['yjs'],+ ignoreDependencies: [+ 'yjs',+ ],},'packages/desktop': {entry: [diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.tsindex a50b00123..4549c74b3 100644--- a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts+++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts@@ -1,3 +1,4 @@+import { readFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts b/packages/app/tests/integration/indented-jsx-byte-sacred.test.tsindex 6fcdc23f2..4fc2c11f9 100644--- a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts+++ b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts@@ -1,3 +1,4 @@+import { setTimeout as wait } from 'node:timers/promises';import { normalizeBridge } from '@inkeep/open-knowledge-core';import { updateYFragment } from '@tiptap/y-tiptap';diff --git a/packages/app/tests/integration/interior-edit-freshness.test.ts b/packages/app/tests/integration/interior-edit-freshness.test.tsindex 40b5cdfd6..9f2f6feb6 100644--- a/packages/app/tests/integration/interior-edit-freshness.test.ts+++ b/packages/app/tests/integration/interior-edit-freshness.test.ts@@ -1,3 +1,4 @@+import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts b/packages/app/tests/integration/keystroke-cadence-danger-space.test.tsindex 93666af70..2a085742b 100644--- a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts+++ b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts@@ -1,3 +1,4 @@+import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/producer-guard-wiring.test.ts b/packages/app/tests/integration/producer-guard-wiring.test.tsindex 867598d34..878d5196b 100644--- a/packages/app/tests/integration/producer-guard-wiring.test.ts+++ b/packages/app/tests/integration/producer-guard-wiring.test.ts@@ -1,3 +1,4 @@+import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.tsindex 0970ac12f..4ada3cf26 100644--- a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts+++ b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts@@ -1,3 +1,4 @@+import { randomUUID } from 'node:crypto';import { afterEach, describe, expect, test } from 'vitest';import { agentWriteMd, createTestServer, type TestServer } from './test-harness';diff --git a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts b/packages/app/tests/integration/qa-045-restore-after-gc.test.tsindex b632185e1..cbe2b705f 100644--- a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts+++ b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts@@ -1,3 +1,4 @@+import { execFileSync } from 'node:child_process';import { randomUUID } from 'node:crypto';import { join } from 'node:path';diff --git a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.tsindex c530ce4ba..89bc9b4d9 100644--- a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts+++ b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts@@ -1,3 +1,4 @@+import { randomUUID } from 'node:crypto';import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';import type * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-after-rollback.test.ts b/packages/app/tests/integration/undo-after-rollback.test.tsindex d3767adc9..a78aeb0e4 100644--- a/packages/app/tests/integration/undo-after-rollback.test.ts+++ b/packages/app/tests/integration/undo-after-rollback.test.ts@@ -1,3 +1,4 @@+import { ROLLBACK_ORIGIN } from '@inkeep/open-knowledge-server';import { afterAll, beforeAll, describe, expect, test } from 'vitest';import * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-recycle-reset.test.ts b/packages/app/tests/integration/undo-recycle-reset.test.tsindex b076f955c..e7cc69c3f 100644--- a/packages/app/tests/integration/undo-recycle-reset.test.ts+++ b/packages/app/tests/integration/undo-recycle-reset.test.ts@@ -1,3 +1,4 @@+import { writeFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/undo-rederive-survival.test.ts b/packages/app/tests/integration/undo-rederive-survival.test.tsindex 7935e35f8..9ddcd0730 100644--- a/packages/app/tests/integration/undo-rederive-survival.test.ts+++ b/packages/app/tests/integration/undo-rederive-survival.test.ts@@ -1,3 +1,4 @@+import { prependFrontmatter } from '@inkeep/open-knowledge-core';import { Plugin, PluginKey } from '@tiptap/pm/state';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.tsindex e44c217cb..a3aa28d78 100644--- a/packages/cli/src/cli.ts+++ b/packages/cli/src/cli.ts@@ -19,7 +19,12 @@ import {formatIgnoredCommittedKey,humanFormat,} from '@inkeep/open-knowledge-core';-import { type Config, ConfigSchema, trustSystemCertificates } from '@inkeep/open-knowledge-server';+import {+ type Config,+ ConfigSchema,+ loggerFactory,+ trustSystemCertificates,+} from '@inkeep/open-knowledge-server';import { Command } from 'commander';import { getCliLogger, initCliLogger } from './cli-logger.ts';import { auditCommand } from './commands/audit.ts';@@ -77,7 +82,7 @@ program).option('--no-color', 'Disable color output').option('--color', 'Force color output')- .hook('preAction', (thisCommand) => {+ .hook('preAction', (thisCommand, actionCommand) => {const opts = thisCommand.opts();const cwd = opts.cwd as string | undefined;if (cwd !== undefined) {@@ -90,6 +95,12 @@ programprocess.env.OK_CONSOLE_LEVEL = level;}+ const actionOpts = actionCommand.opts<{ json?: boolean }>();+ const actionName = actionCommand.name();+ if ((actionName === 'clone' || actionName === 'init') && actionOpts.json === true) {+ loggerFactory.configure({ pinoConfig: { consoleDestination: 2 } });+ }+const subcommandName = thisCommand.args?.[0];const anchorRoot = resolveProjectAnchor(subcommandName, process.cwd());if (anchorRoot !== null) {diff --git a/packages/cli/tests/e2e/cli-linux-e2e.ts b/packages/cli/tests/e2e/cli-linux-e2e.tsindex 033ad0602..27b635f42 100644--- a/packages/cli/tests/e2e/cli-linux-e2e.ts+++ b/packages/cli/tests/e2e/cli-linux-e2e.ts@@ -1,5 +1,6 @@import { type ChildProcess, execFileSync, spawn, spawnSync } from 'node:child_process';import {+ chmodSync,existsSync,mkdirSync,mkdtempSync,@@ -15,7 +16,7 @@ import { fileURLToPath } from 'node:url';import { getMachineId } from '@inkeep/open-knowledge-server';import { Client } from '@modelcontextprotocol/sdk/client/index.js';import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';-import { afterAll, beforeAll, describe, expect, test } from 'vitest';+import { afterAll, beforeAll, describe, expect, onTestFinished, test } from 'vitest';import {legacyServerLock,lockWrittenBeforeProcessStarted,@@ -23,6 +24,10 @@ import {startOwnedProcess,stopOwnedProcess,} from '../../../../test-support/owned-process.test-helper.ts';+import {+ type CommandResult,+ spawnUnderOwnerLifeline,+} from '../../../../test-support/owner-lifeline.ts';import { installPackedCli } from './packed-install.test-helper';const HERE = dirname(fileURLToPath(import.meta.url));@@ -63,10 +68,17 @@ function progressOf(acquisition: Acquisition, status: string) {return acquisition?.progress.get(status) ?? new Set<string>();}+interface CliRunResult {+ status: number | null;+ signal: NodeJS.Signals | null;+ stdout: string;+ stderr: string;+}+function runOk(args: string[],opts: { cwd?: string; timeoutMs?: number; env?: NodeJS.ProcessEnv } = {},-): { status: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string } {+): CliRunResult {const r = spawnSync(NODE, [H.cliPath, ...args], {cwd: opts.cwd ?? H.contentDir,timeout: opts.timeoutMs ?? 30_000,@@ -125,6 +137,106 @@ function readLockPort(): number | null {}}+const SERVER_INFO_RECORD = /^\[\d{2}:\d{2}:\d{2}\] INFO \((?:project-git|skills)\): /m;++function outputOf(result: CliRunResult): string {+ return `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`;+}++function isolatedCliRunner(prefix: string) {+ const root = realpathSync(mkdtempSync(join(tmpdir(), prefix)));+ const fakeHome = join(root, 'home');+ const gitHome = join(root, 'git-home');+ let ownedRun: ReturnType<typeof spawnUnderOwnerLifeline> | undefined;+ let outputClosed: Promise<void> = Promise.resolve();+ let commandResult: Promise<CommandResult | null> = Promise.resolve(null);+ onTestFinished(async () => {+ if (+ ownedRun !== undefined &&+ ownedRun.child.exitCode === null &&+ ownedRun.child.signalCode === null &&+ !ownedRun.child.stdin.destroyed+ ) {+ ownedRun.child.stdin.end();+ }+ await outputClosed;+ await commandResult;+ rmSync(root, { recursive: true, force: true });+ });+ mkdirSync(fakeHome, { recursive: true });+ mkdirSync(gitHome, { recursive: true });+ const fixtureGitEnv: NodeJS.ProcessEnv = {+ ...Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_'))),+ HOME: gitHome,+ GIT_CONFIG_GLOBAL: '/dev/null',+ GIT_CONFIG_SYSTEM: '/dev/null',+ GIT_CONFIG_NOSYSTEM: '1',+ };+ const fixtureGit = (args: string[], cwd = root) => {+ execFileSync('git', args, { cwd, env: fixtureGitEnv, stdio: 'ignore' });+ };+ const ghShim = join(root, 'gh');+ writeFileSync(ghShim, '#!/bin/sh\nprintf token-for-e2e\n', 'utf8');+ chmodSync(ghShim, 0o755);+ mkdirSync(join(fakeHome, '.ok'), { recursive: true });+ writeFileSync(+ join(fakeHome, '.ok', 'skill-state.yml'),+ 'schema: 1\nbundles:\n open-knowledge-discovery:\n enabled: true\n decidedAt: 2026-01-01T00:00:00.000Z\n',+ 'utf8',+ );++ const runCli = async (+ args: string[],+ cwd: string,+ scenarioEnv: NodeJS.ProcessEnv = {},+ ): Promise<CliRunResult> => {+ const owned = spawnUnderOwnerLifeline(NODE, [H.cliPath, '--log-level', 'info', ...args], {+ cwd,+ env: {+ ...Object.fromEntries(+ Object.entries(fixtureGitEnv).filter(([key]) => !key.startsWith('GIT_')),+ ),+ HOME: fakeHome,+ XDG_CONFIG_HOME: join(fakeHome, '.config'),+ PATH: `${root}:${fixtureGitEnv.PATH ?? ''}`,+ DBUS_SESSION_BUS_ADDRESS: '',+ XDG_RUNTIME_DIR: '',+ DO_NOT_TRACK: '1',+ LOG_LEVEL: 'info',+ OK_CONSOLE_LEVEL: 'info',+ GIT_CONFIG_NOSYSTEM: '1',+ ...scenarioEnv,+ },+ });+ ownedRun = owned;+ let stdout = '';+ let stderr = '';+ owned.child.stdout.setEncoding('utf8').on('data', (chunk) => {+ stdout += chunk;+ });+ owned.child.stderr.setEncoding('utf8').on('data', (chunk) => {+ stderr += chunk;+ });+ outputClosed = new Promise<void>((resolve, reject) => {+ owned.child.once('error', reject);+ owned.child.once('close', () => resolve());+ });+ commandResult = owned.commandResult;+ await outputClosed;+ const command = await commandResult;+ const result: CliRunResult = {+ status: command?.code ?? null,+ signal: command?.signal ?? null,+ stdout,+ stderr,+ };+ expect(result.status, `signal=${result.signal}\n${outputOf(result)}`).toBe(0);+ return result;+ };++ return { root, fixtureGit, runCli };+}+beforeAll(async () => {if (!existsSync(WORKSPACE_DIST_CLI)) {throw new Error(@@ -221,6 +333,61 @@ describe(`CLI Linux e2e (${SUT_MODE} SUT)`, () => {expect(existsSync(join(H.contentDir, '.ok'))).toBe(true);});+ test('2c. ok clone --json keeps fresh clone stdout parseable as JSONL', async () => {+ const isolated = isolatedCliRunner('ok-clone-jsonl-fresh-');+ const run = join(isolated.root, 'run');+ mkdirSync(run, { recursive: true });+ isolated.fixtureGit(['init', '-q']);+ isolated.fixtureGit(['config', 'maintenance.auto', 'false']);+ isolated.fixtureGit(['config', 'user.email', '[email protected]']);+ isolated.fixtureGit(['config', 'user.name', 'OK E2E']);++ const source = join(isolated.root, 'source');+ const bare = join(isolated.root, 'remote.git');+ const cloneUrl = 'https://127.0.0.1/owner/repo';+ mkdirSync(source, { recursive: true });+ isolated.fixtureGit(['init', '-q', '--initial-branch=main'], source);+ isolated.fixtureGit(['config', 'maintenance.auto', 'false'], source);+ isolated.fixtureGit(['config', 'user.email', '[email protected]'], source);+ isolated.fixtureGit(['config', 'user.name', 'OK E2E'], source);+ writeFileSync(join(source, 'README.md'), '# Clone fixture\n', 'utf8');+ isolated.fixtureGit(['add', 'README.md'], source);+ isolated.fixtureGit(['commit', '-q', '-m', 'fixture'], source);+ isolated.fixtureGit(['-c', 'maintenance.auto=false', 'clone', '-q', '--bare', source, bare]);+ isolated.fixtureGit(['config', 'maintenance.auto', 'false'], bare);++ const result = await isolated.runCli(['clone', cloneUrl, '../clone', '--json'], run, {+ GIT_CONFIG_COUNT: '1',+ GIT_CONFIG_KEY_0: `url.file://${bare}.insteadOf`,+ GIT_CONFIG_VALUE_0: cloneUrl,+ });++ expect(result.stderr, outputOf(result)).toMatch(SERVER_INFO_RECORD);+ const lines = result.stdout.endsWith('\n')+ ? result.stdout.slice(0, -1).split('\n')+ : result.stdout.split('\n');+ expect(lines.length).toBeGreaterThan(0);+ for (const [index, line] of lines.entries()) {+ expect(+ () => JSON.parse(line),+ `stdout line ${index + 1}: ${line}\n${outputOf(result)}`,+ ).not.toThrow();+ }+ });++ test('2d. ok init --no-mcp --json keeps stdout parseable as JSON', async () => {+ const isolated = isolatedCliRunner('ok-init-json-');++ const result = await isolated.runCli(['init', '--no-mcp', '--json'], isolated.root);++ expect(result.stderr, outputOf(result)).toMatch(SERVER_INFO_RECORD);+ let report: { projectRoot?: string } | undefined;+ expect(() => {+ report = JSON.parse(result.stdout) as { projectRoot?: string };+ }, outputOf(result)).not.toThrow();+ expect(report?.projectRoot).toBe(isolated.root);+ });+test('3. ok start boots, writes server.lock with a port, and serves HTTP', async () => {H.server = spawn(NODE, [H.cliPath, 'start', '--port', String(START_PORT)], {cwd: H.contentDir,diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.tsindex f4cc8a286..2a597ec3d 100644--- a/packages/core/src/bridge/compose.ts+++ b/packages/core/src/bridge/compose.ts@@ -1,3 +1,4 @@+import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts';export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved';diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.tsindex e7e63f43d..ffd8fce06 100644--- a/packages/core/src/bridge/doc-boundary-space.ts+++ b/packages/core/src/bridge/doc-boundary-space.ts@@ -1,3 +1,4 @@+import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.tsindex fc267cbea..fb0ba5a7c 100644--- a/packages/core/src/markdown/comment-promoter.ts+++ b/packages/core/src/markdown/comment-promoter.ts@@ -1,3 +1,4 @@+import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.tsindex 861003fb4..ae8cb390b 100644--- a/packages/core/src/markdown/merged-walker.ts+++ b/packages/core/src/markdown/merged-walker.ts@@ -1,3 +1,4 @@+import type { Nodes, Parent, Root } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.tsindex 4d3250d3a..8be632ba4 100644--- a/packages/core/src/markdown/safe-url.ts+++ b/packages/core/src/markdown/safe-url.ts@@ -1,3 +1,4 @@+export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');diff --git a/packages/server/src/logger.ts b/packages/server/src/logger.tsindex ecaef1156..b06da516e 100644--- a/packages/server/src/logger.ts+++ b/packages/server/src/logger.ts@@ -15,11 +15,12 @@ function otelMixin(): Record<string, unknown> {};}-function shouldColorize(): boolean {+function shouldColorize(destination: 1 | 2): boolean {if (process.env.NO_COLOR && process.env.NO_COLOR !== '') {return false;}- return process.stdout.isTTY ?? false;+ const stream = destination === 2 ? process.stderr : process.stdout;+ return stream.isTTY ?? false;}const VALID_LOG_LEVELS = new Set(['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent']);@@ -38,6 +39,7 @@ export interface PinoLoggerConfig {transportConfigs?: TransportSingleOptions[];fileSink?: PinoFileSinkOpts;redactPaths?: readonly string[];+ consoleDestination?: 1 | 2;}export class PinoLogger {@@ -45,6 +47,7 @@ export class PinoLogger {private transportConfigs: TransportSingleOptions[] = [];private fileSinkOpts: PinoFileSinkOpts | undefined;private redactPaths: readonly string[] | undefined;+ private consoleDestination: 1 | 2 = 1;private activeFileSink: PinoFileSink | undefined;private pinoInstance: PinoLoggerInstance;private options: LoggerOptions;@@ -69,6 +72,9 @@ export class PinoLogger {if (config.fileSink) {this.fileSinkOpts = config.fileSink;}+ if (config.consoleDestination !== undefined) {+ this.consoleDestination = config.consoleDestination;+ }if (config.redactPaths && config.redactPaths.length > 0) {this.redactPaths = config.redactPaths;this.options = {@@ -110,9 +116,10 @@ export class PinoLogger {let prettyStream: ReturnType<typeof pinoPretty>;try {prettyStream = pinoPretty({- colorize: shouldColorize(),+ colorize: shouldColorize(this.consoleDestination),translateTime: 'HH:MM:ss',ignore: 'pid,hostname',+ destination: this.consoleDestination,});} catch (err) {console.warn('[PinoLogger] pino-pretty failed, falling back to JSON:', err);@@ -124,7 +131,10 @@ export class PinoLogger {pino.multistream([{ stream: fileSink, level: fileLevel }]),);}- return pino({ ...this.options, level: consoleLevel });+ return pino(+ { ...this.options, level: consoleLevel },+ pino.destination(this.consoleDestination),+ );}if (this.fileSinkOpts) {@@ -179,6 +189,9 @@ export class PinoLogger {if (config.fileSink) {this.fileSinkOpts = config.fileSink;}+ if (config.consoleDestination !== undefined) {+ this.consoleDestination = config.consoleDestination;+ }if (config.redactPaths && config.redactPaths.length > 0) {this.redactPaths = config.redactPaths;this.options = {