Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Trace sync credential test phases on failure (diagnostics only) (#5584)
Nick Gomez49640e7aea61
diff --git a/knip.config.ts b/knip.config.tsindex d3638888e..65f0e8807 100644--- a/knip.config.ts+++ b/knip.config.ts@@ -147,15 +147,11 @@ export default {'packages/server': {entry: ['src/**/*.test.ts', 'src/parse-worker.ts'],project: 'src/**',- ignoreDependencies: [- '@types/shell-quote',- ],+ ignoreDependencies: ['@types/shell-quote'],},'packages/cli': {entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'],- ignoreDependencies: [- 'yjs',- ],+ ignoreDependencies: ['yjs'],},'packages/desktop': {entry: [diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.tsindex a28cc2b18..8a60e5826 100644--- a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts+++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts@@ -1,4 +1,3 @@-import { readFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts b/packages/app/tests/integration/indented-jsx-byte-sacred.test.tsindex 4fc2c11f9..6fcdc23f2 100644--- a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts+++ b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { normalizeBridge } from '@inkeep/open-knowledge-core';import { updateYFragment } from '@tiptap/y-tiptap';diff --git a/packages/app/tests/integration/interior-edit-freshness.test.ts b/packages/app/tests/integration/interior-edit-freshness.test.tsindex 9f2f6feb6..40b5cdfd6 100644--- a/packages/app/tests/integration/interior-edit-freshness.test.ts+++ b/packages/app/tests/integration/interior-edit-freshness.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts b/packages/app/tests/integration/keystroke-cadence-danger-space.test.tsindex 2a085742b..93666af70 100644--- a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts+++ b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/producer-guard-wiring.test.ts b/packages/app/tests/integration/producer-guard-wiring.test.tsindex 878d5196b..867598d34 100644--- a/packages/app/tests/integration/producer-guard-wiring.test.ts+++ b/packages/app/tests/integration/producer-guard-wiring.test.ts@@ -1,4 +1,3 @@-import { setTimeout as wait } from 'node:timers/promises';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';import { afterAll, beforeAll, describe, expect, test } from 'vitest';diff --git a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.tsindex 4ada3cf26..0970ac12f 100644--- a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts+++ b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts@@ -1,4 +1,3 @@-import { randomUUID } from 'node:crypto';import { afterEach, describe, expect, test } from 'vitest';import { agentWriteMd, createTestServer, type TestServer } from './test-harness';diff --git a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts b/packages/app/tests/integration/qa-045-restore-after-gc.test.tsindex cbe2b705f..b632185e1 100644--- a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts+++ b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts@@ -1,4 +1,3 @@-import { execFileSync } from 'node:child_process';import { randomUUID } from 'node:crypto';import { join } from 'node:path';diff --git a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.tsindex 89bc9b4d9..c530ce4ba 100644--- a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts+++ b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts@@ -1,4 +1,3 @@-import { randomUUID } from 'node:crypto';import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest';import type * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-after-rollback.test.ts b/packages/app/tests/integration/undo-after-rollback.test.tsindex a78aeb0e4..d3767adc9 100644--- a/packages/app/tests/integration/undo-after-rollback.test.ts+++ b/packages/app/tests/integration/undo-after-rollback.test.ts@@ -1,4 +1,3 @@-import { ROLLBACK_ORIGIN } from '@inkeep/open-knowledge-server';import { afterAll, beforeAll, describe, expect, test } from 'vitest';import * as Y from 'yjs';diff --git a/packages/app/tests/integration/undo-recycle-reset.test.ts b/packages/app/tests/integration/undo-recycle-reset.test.tsindex e7cc69c3f..b076f955c 100644--- a/packages/app/tests/integration/undo-recycle-reset.test.ts+++ b/packages/app/tests/integration/undo-recycle-reset.test.ts@@ -1,4 +1,3 @@-import { writeFileSync } from 'node:fs';import { join } from 'node:path';import { setTimeout as wait } from 'node:timers/promises';diff --git a/packages/app/tests/integration/undo-rederive-survival.test.ts b/packages/app/tests/integration/undo-rederive-survival.test.tsindex 9ddcd0730..7935e35f8 100644--- a/packages/app/tests/integration/undo-rederive-survival.test.ts+++ b/packages/app/tests/integration/undo-rederive-survival.test.ts@@ -1,4 +1,3 @@-import { prependFrontmatter } from '@inkeep/open-knowledge-core';import { Plugin, PluginKey } from '@tiptap/pm/state';import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap';diff --git a/packages/cli/src/commands/git-credential-fixtures.test-helper.ts b/packages/cli/src/commands/git-credential-fixtures.test-helper.tsindex a57147dae..b31f37b0c 100644--- a/packages/cli/src/commands/git-credential-fixtures.test-helper.ts+++ b/packages/cli/src/commands/git-credential-fixtures.test-helper.ts@@ -6,12 +6,23 @@ import { shellSingleQuote } from '@inkeep/open-knowledge-core';const CLI_PACKAGE_DIR = resolve(import.meta.dirname, '../..');-function recordCliCallLines(callLog: string): string[] {+function phaseRecorderLines(phaseLog: string | undefined): string[] {+ return phaseLog === undefined+ ? []+ : [+ `const phase = stage => appendFileSync(${JSON.stringify(phaseLog)}, JSON.stringify({ stage, pid: process.pid, ppid: process.ppid, at: Date.now() }) + '\\n');`,+ ];+}++function recordCliCallLines(callLog: string, phaseLog?: string): string[] {return ["import { spawnSync } from 'node:child_process';","import { appendFileSync, readFileSync } from 'node:fs';",+ ...phaseRecorderLines(phaseLog),+ ...(phaseLog === undefined ? [] : ["phase('wrapper:loaded');"]),'const args = process.argv.slice(2);',"const input = readFileSync(0, 'utf-8');",+ ...(phaseLog === undefined ? [] : ["phase('wrapper:input-ended');"]),'const fields = {};',"for (const line of input.split('\\n')) {"," const at = line.indexOf('=');",@@ -79,6 +90,7 @@ export function writeCliCredentialStandIn(options: {dir: string;callLog: string;authFile: string;+ phaseLog?: string;}): string {const runner = join(options.dir, 'ok-git-credential-get.mts');const moduleUrl = (path: string) =>@@ -88,8 +100,18 @@ export function writeCliCredentialStandIn(options: {[`import { FileBackend } from ${moduleUrl('src/auth/token-store.ts')};`,`import { handleCredentialGet } from ${moduleUrl('src/commands/auth/git-credential-get.ts')};`,+ ...(options.phaseLog === undefined+ ? []+ : [+ "import { appendFileSync } from 'node:fs';",+ ...phaseRecorderLines(options.phaseLog),+ "phase('module-loaded');",+ "process.stdin.once('end', () => phase('input-ended'));",+ ]),`const store = new FileBackend(${JSON.stringify(options.authFile)});`,- 'process.exit(await handleCredentialGet(process.stdin, process.stdout, store));',+ 'const code = await handleCredentialGet(process.stdin, process.stdout, store);',+ ...(options.phaseLog === undefined ? [] : ["phase('handler-returned');"]),+ 'process.exit(code);','',].join('\n'),'utf-8',@@ -98,9 +120,11 @@ export function writeCliCredentialStandIn(options: {writeFileSync(entry,[- ...recordCliCallLines(options.callLog),+ ...recordCliCallLines(options.callLog, options.phaseLog),"if (args.join(' ') !== 'auth git-credential get') process.exit(1);",+ ...(options.phaseLog === undefined ? [] : ["phase('helper:start');"]),`const helper = spawnSync(process.execPath, ['--conditions=development', '--import', 'tsx', ${JSON.stringify(runner)}], { cwd: ${JSON.stringify(CLI_PACKAGE_DIR)}, input, stdio: ['pipe', 'inherit', 'inherit'] });`,+ ...(options.phaseLog === undefined ? [] : ["phase('helper:returned');"]),'process.exit(helper.status ?? 1);','',].join('\n'),diff --git a/packages/cli/src/commands/github-stand-in.test-helper.ts b/packages/cli/src/commands/github-stand-in.test-helper.tsindex 96cdb7f00..2859d0052 100644--- a/packages/cli/src/commands/github-stand-in.test-helper.ts+++ b/packages/cli/src/commands/github-stand-in.test-helper.ts@@ -213,6 +213,7 @@ function serveThroughHttpBackend(user: string,repositoriesRoot: string,settle: (status: number) => void,+ onPhase?: (phase: string) => void,): void {const env: Record<string, string> = {...hermeticGitEnv(repositoriesRoot),@@ -233,7 +234,11 @@ function serveThroughHttpBackend(const value = req.headers[header];if (typeof value === 'string') env[variable] = value;}+ onPhase?.('backend:start');const backend = spawn('git', ['http-backend'], { env, stdio: ['pipe', 'pipe', 'ignore'] });+ backend.stdin.once('finish', () => onPhase?.('backend:stdin-finished'));+ backend.once('exit', () => onPhase?.('backend:exited'));+ backend.stdout.once('end', () => onPhase?.('backend:stdout-ended'));req.pipe(backend.stdin);let pending = Buffer.alloc(0);let headersSent = false;@@ -262,6 +267,7 @@ function serveThroughHttpBackend(if (body.length > 0) res.write(body);});backend.on('close', () => {+ onPhase?.('backend:closed');if (!headersSent) {settle(500);res.writeHead(500);@@ -276,6 +282,7 @@ export async function startGitHubStandIn(options: {deniedPassword?: string;repositories: string[];enterpriseHosts?: Record<string, string[]>;+ onPhase?: (phase: string, requestId?: number) => void;}): Promise<GitHubStandIn> {const hosts = new Map<string, StandInHost>();for (const [host, acceptedPasswords] of [@@ -323,9 +330,16 @@ export async function startGitHubStandIn(options: {const caFile = join(options.root, 'github-stand-in-ca.pem');writeFileSync(caFile, certificates.caPem, 'utf8');+ let requestId = 0;const tlsServer = createHttpsServer({ key: certificates.serverKeyPem, cert: certificates.serverCertPem },(req, res) => {+ const id = ++requestId;+ const phase = (stage: string) => options.onPhase?.(stage, id);+ phase('request:received');+ req.once('end', () => phase('request:ended'));+ res.once('finish', () => phase('response:finished'));+ res.once('close', () => phase('response:closed'));const host = String(req.headers.host ?? '').replace(/:\d+$/, '').toLowerCase();@@ -368,15 +382,18 @@ export async function startGitHubStandIn(options: {(status) => {request.status = status;},+ phase,);},);+ tlsServer.on('secureConnection', () => options.onPhase?.('tls:ready'));const proxy = createHttpServer((req, res) => {req.resume();res.writeHead(403);res.end();});proxy.on('connect', (req: IncomingMessage, socket: Duplex, head: Buffer) => {+ options.onPhase?.('proxy:connected');const target = (req.url ?? '').toLowerCase();if (!/:\d+$/.test(target) || !hosts.has(target.replace(/:\d+$/, ''))) {socket.end('HTTP/1.1 403 Forbidden\r\n\r\n');diff --git a/packages/cli/src/commands/sync-credential-helper.test.ts b/packages/cli/src/commands/sync-credential-helper.test.tsindex 78536ca4e..9dc8b7bfc 100644--- a/packages/cli/src/commands/sync-credential-helper.test.ts+++ b/packages/cli/src/commands/sync-credential-helper.test.ts@@ -1,11 +1,13 @@import { execFileSync } from 'node:child_process';import {copyFileSync,+ existsSync,mkdirSync,mkdtempSync,readFileSync,realpathSync,rmSync,+ statSync,writeFileSync,} from 'node:fs';import { tmpdir } from 'node:os';@@ -78,6 +80,28 @@ interface Scenario {const ghUnavailable = (): GhDetectResult => ({ available: false });+function readDiagnosticLines<T>(file: string): Array<T | 'unparseable'> {+ if (!existsSync(file)) return [];+ return readFileSync(file, 'utf-8')+ .split('\n')+ .filter((line) => line !== '')+ .map((line) => {+ try {+ return JSON.parse(line) as T;+ } catch {+ return 'unparseable';+ }+ });+}++function countLines(file: string): number | null {+ return existsSync(file) ? readFileSync(file, 'utf-8').split('\n').length - 1 : null;+}++function modifiedAt(file: string): number | null {+ return existsSync(file) ? Math.trunc(statSync(file).mtimeMs) : null;+}+function originUpstreamOnGitHub(git: GitRunner, server: GitHubStandIn): void {git('remote', 'add', 'origin', ORIGIN_URL);git('push', server.repositoryDir('alice/demo.git'), 'main:main');@@ -118,6 +142,8 @@ describe("ok sync without a running server authenticates through OpenKnowledge'slet traceFile: string;let authFile: string;let cliStandIn: string;+ let helperPhases: string;+ let phases: Array<{ stage: string; at: number; requestId?: number }>;let standIn: GitHubStandIn | undefined;let interceptor: PlainHttpInterceptor | undefined;@@ -137,6 +163,8 @@ describe("ok sync without a running server authenticates through OpenKnowledge'scliHelperLog = join(workspace, 'cli-helper-calls.jsonl');traceFile = join(workspace, 'git-trace.log');authFile = join(workspace, 'ok-auth.yml');+ helperPhases = join(workspace, 'helper-phases.jsonl');+ phases = [{ stage: 'setup:start', at: Date.now() }];vi.stubEnv('HOME', home);vi.stubEnv('USERPROFILE', home);vi.stubEnv('XDG_CONFIG_HOME', join(home, '.config'));@@ -154,17 +182,41 @@ describe("ok sync without a running server authenticates through OpenKnowledge's}mkdirSync(home, { recursive: true });mkdirSync(projectDir, { recursive: true });- cliStandIn = writeCliCredentialStandIn({ dir: workspace, callLog: cliHelperLog, authFile });+ cliStandIn = writeCliCredentialStandIn({+ dir: workspace,+ callLog: cliHelperLog,+ authFile,+ phaseLog: helperPhases,+ });});- afterEach(async () => {+ afterEach(async (context) => {vi.restoreAllMocks();vi.unstubAllEnvs();- await standIn?.close();- standIn = undefined;- await interceptor?.close();- interceptor = undefined;- rmSync(workspace, { recursive: true, force: true });+ try {+ if (context.task.result?.state === 'fail') {+ process.stderr.write(+ `${JSON.stringify({+ diagnostic: 'sync-credential-phases',+ phases,+ helperPhases: readDiagnosticLines(helperPhases),+ helperCalls: readDiagnosticLines<CliHelperCall>(cliHelperLog).map((call) =>+ call === 'unparseable' ? call : call.args.at(-1),+ ),+ requests: standIn?.requests.map(({ method, status }) => ({ method, status })),+ gitTracePresent: existsSync(traceFile),+ gitTraceLines: countLines(traceFile),+ gitTraceModifiedAt: modifiedAt(traceFile),+ })}\n`,+ );+ }+ } finally {+ await standIn?.close();+ standIn = undefined;+ await interceptor?.close();+ interceptor = undefined;+ rmSync(workspace, { recursive: true, force: true });+ }});function gitCredential(@@ -179,8 +231,11 @@ describe("ok sync without a running server authenticates through OpenKnowledge's}async function arrange(scenario: Scenario): Promise<GitHubStandIn> {+ const currentPhases = phases;+ currentPhases.push({ stage: 'arrange:server-start', at: Date.now() });const started = await startGitHubStandIn({root: join(workspace, 'github'),+ onPhase: (stage, requestId) => currentPhases.push({ stage, requestId, at: Date.now() }),acceptedPassword: scenario.gitHubAccepts,...(scenario.gitHubDenies === undefined ? {} : { deniedPassword: scenario.gitHubDenies }),repositories: ['alice/demo.git'],@@ -197,6 +252,7 @@ describe("ok sync without a running server authenticates through OpenKnowledge's},});standIn = started;+ currentPhases.push({ stage: 'arrange:server-ready', at: Date.now() });const recordingHelper = writeRecordingCredentialHelper(workspace, helperLog);const globalConfig = join(home, '.gitconfig');for (const [key, value] of [@@ -238,6 +294,7 @@ describe("ok sync without a running server authenticates through OpenKnowledge's(scenario.repository ?? originUpstreamOnGitHub)(git, started);git('commit', '--allow-empty', '-m', 'local work to push');+ currentPhases.push({ stage: 'arrange:repository-ready', at: Date.now() });const liveness = scenario.ambientCredential ?? {username: 'liveness-probe',password: 'ok-test-sync-liveness-probe-password',@@ -266,6 +323,7 @@ describe("ok sync without a running server authenticates through OpenKnowledge's}writeFileSync(helperLog, '', 'utf-8');vi.stubEnv('GIT_TRACE', traceFile);+ currentPhases.push({ stage: 'arrange:ready', at: Date.now() });return started;}@@ -276,17 +334,38 @@ describe("ok sync without a running server authenticates through OpenKnowledge's): Promise<{ events: SyncEvent[]; stderr: string; error: unknown }> {expect(readServerLock(resolveLockDir(projectDir))).toBeNull();const events: SyncEvent[] = [];+ const currentPhases = phases;+ currentPhases.push({ stage: 'sync:start', at: Date.now() });let stderr = '';const writes = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {for (const line of String(chunk).split('\n')) {- if (line.trim() !== '') events.push(JSON.parse(line) as SyncEvent);+ if (line.trim() !== '') {+ const event = JSON.parse(line) as SyncEvent;+ events.push(event);+ if (event.type === 'step' && (event.step === 'pull' || event.step === 'push'))+ currentPhases.push({ stage: `sync:${event.step}`, at: Date.now() });+ if (event.type === 'pull' || event.type === 'push')+ currentPhases.push({ stage: `sync:${event.type}-finished`, at: Date.now() });+ }}return true;});const errorWrites =output === 'text'? vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => {- stderr += String(chunk);+ const text = String(chunk);+ stderr += text;+ if (text === `Running ${op} directly (no live server)\n`)+ currentPhases.push({+ stage: `sync:${op === 'push' ? 'push' : 'pull'}`,+ at: Date.now(),+ });+ if (text.startsWith(' pull: ')) {+ currentPhases.push({ stage: 'sync:pull-finished', at: Date.now() });+ if (op === 'sync') currentPhases.push({ stage: 'sync:push', at: Date.now() });+ }+ if (text === ' push: ok\n')+ currentPhases.push({ stage: 'sync:push-finished', at: Date.now() });return true;}): undefined;@@ -300,7 +379,9 @@ describe("ok sync without a running server authenticates through OpenKnowledge'slet error: unknown = null;try {await runSync(options, {} as Config, projectDir);+ currentPhases.push({ stage: 'sync:returned', at: Date.now() });} catch (caught) {+ currentPhases.push({ stage: 'sync:rejected', at: Date.now() });error = caught;} finally {process.argv[1] = cliEntry;diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.tsindex 2a597ec3d..f4cc8a286 100644--- a/packages/core/src/bridge/compose.ts+++ b/packages/core/src/bridge/compose.ts@@ -1,4 +1,3 @@-import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts';export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved';diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.tsindex 01ef76a9f..8fb428b90 100644--- a/packages/core/src/bridge/doc-boundary-space.ts+++ b/packages/core/src/bridge/doc-boundary-space.ts@@ -1,4 +1,3 @@-import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts';import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts';diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.tsindex fb0ba5a7c..fc267cbea 100644--- a/packages/core/src/markdown/comment-promoter.ts+++ b/packages/core/src/markdown/comment-promoter.ts@@ -1,4 +1,3 @@-import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.tsindex fa3cfcd86..bfd7e3109 100644--- a/packages/core/src/markdown/merged-walker.ts+++ b/packages/core/src/markdown/merged-walker.ts@@ -1,4 +1,3 @@-import type { Nodes, Parent, Root } from 'mdast';import { SKIP, visit } from 'unist-util-visit';import type { VFile } from 'vfile';diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.tsindex 8be632ba4..4d3250d3a 100644--- a/packages/core/src/markdown/safe-url.ts+++ b/packages/core/src/markdown/safe-url.ts@@ -1,4 +1,3 @@-export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const;const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');