Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(open-knowledge): NUL-safe git path helper, fix non-ASCII renames (#2466)

miles-kt-inkeep23ee122671db
diff --git a/.changeset/consolidate-git-path-parsing.md b/.changeset/consolidate-git-path-parsing.md
new file mode 100644
index 000000000..0a288a5fc
--- /dev/null
+++ b/.changeset/consolidate-git-path-parsing.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Fix git rename detection for non-ASCII filenames in two more places. A shared doc or folder whose name contains non-ASCII characters (umlauts, accents, CJK) that is renamed on the remote now reports its correct new location instead of showing as deleted, and version-timeline entries recover the real document name instead of garbled text. All git path-listing now routes through a single NUL-safe helper (`git-paths.ts`) so this class of bug cannot recur at new call sites.
diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.ts
index adaa90a19..95082dc4a 100644
--- a/packages/server/src/boot.ts
+++ b/packages/server/src/boot.ts
@@ -43,7 +43,7 @@ import type { Config } from './config/schema.ts';
import { ConflictStore } from './conflict-storage.ts';
import { stripDocExtension } from './doc-extensions.ts';
import { normalizeFsPath } from './fs-traced.ts';
-import { splitNulSeparatedPaths } from './git-handle.ts';
+import { listNames } from './git-paths.ts';
import {
assertGitAvailable,
type GitDetected,
@@ -1118,8 +1118,7 @@ export async function restoreLifecycleFromConflictsJson(args: {
return;
}
const pg = simpleGit({ baseDir: projectDir, timeout: { block: 5_000 } });
- const out = await pg.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
- stillUnmerged = new Set(splitNulSeparatedPaths(out));
+ stillUnmerged = new Set(await listNames(pg, ['diff', '--name-only', '--diff-filter=U']));
} catch (err) {
// Probe failed — fall through and restore everything; the sync
// engine's own reconcile on `start()` will mop up any stragglers.
diff --git a/packages/server/src/conflict-storage.ts b/packages/server/src/conflict-storage.ts
index 1ec0eb14c..8c4d1cf1f 100644
--- a/packages/server/src/conflict-storage.ts
+++ b/packages/server/src/conflict-storage.ts
@@ -13,6 +13,7 @@
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { getLocalDir } from './config/paths.ts';
+import { listNames } from './git-paths.ts';
import { getLogger } from './logger.ts';
import { isWithinDir } from './path-utils.ts';
@@ -255,9 +256,10 @@ export class ConflictStore {
const detectedAt = new Date().toISOString();
let reAdded = false;
try {
- const { splitNulSeparatedPaths } = await import('./git-handle.ts');
- const raw = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
- const unmerged = splitNulSeparatedPaths(raw);
+ // git-paths.ts has no runtime simple-git dependency, so this needs no
+ // dynamic import (unlike createGitInstance above) to keep simple-git
+ // out of the CRUD-test module graph.
+ const unmerged = await listNames(handle.git, ['diff', '--name-only', '--diff-filter=U']);
for (const f of unmerged) {
this.addConflict({ file: f, detectedAt });
}
diff --git a/packages/server/src/git-dirty.ts b/packages/server/src/git-dirty.ts
index 014ad6236..7af7f9a6f 100644
--- a/packages/server/src/git-dirty.ts
+++ b/packages/server/src/git-dirty.ts
@@ -9,27 +9,14 @@
* them.
*/
-import { createGitInstance, splitNulSeparatedPaths } from './git-handle.ts';
+import { createGitInstance } from './git-handle.ts';
+import { listNames, listPorcelainPaths } from './git-paths.ts';
export interface DirtyOverlapResult {
conflicts: boolean;
files: string[];
}
-function parsePorcelainPaths(porcelain: string): string[] {
- // -z porcelain: NUL-separated "XY PATH" records; a rename/copy emits the
- // original path as the following record, without a status prefix.
- const paths: string[] = [];
- const records = porcelain.split('\0');
- for (let i = 0; i < records.length; i++) {
- const record = records[i] ?? '';
- if (record.length < 4) continue;
- paths.push(record.slice(3));
- if (record[0] === 'R' || record[0] === 'C') i++;
- }
- return paths;
-}
-
/**
* Returns the intersection of dirty working-tree files and files changed by
* switching from the current ref to `targetRef`.
@@ -51,19 +38,18 @@ export async function dirtyFilesOverlapWith(
): Promise<DirtyOverlapResult> {
const { git } = createGitInstance(cwd);
- const [porcelain, diff] = await Promise.all([
- git.raw(['status', '--porcelain', '-z']),
+ const [dirtyList, changed] = await Promise.all([
+ listPorcelainPaths(git),
// Two-dot diff — all files differing between HEAD and targetRef, in either
// direction. Three-dot (`HEAD...targetRef`) resolves to merge-base..targetRef
// and misses files HEAD changed since divergence, which `git checkout` would
// still restore to the target's view.
- git.raw(['diff', '--name-only', '-z', `HEAD..${targetRef}`]),
+ listNames(git, ['diff', '--name-only', `HEAD..${targetRef}`]),
]);
- const dirty = new Set(parsePorcelainPaths(porcelain));
+ const dirty = new Set(dirtyList);
if (dirty.size === 0) return { conflicts: false, files: [] };
- const changed = splitNulSeparatedPaths(diff);
if (changed.length === 0) return { conflicts: false, files: [] };
const overlap = new Set<string>();
diff --git a/packages/server/src/git-handle.ts b/packages/server/src/git-handle.ts
index 8feacb222..b3ab8f9b7 100644
--- a/packages/server/src/git-handle.ts
+++ b/packages/server/src/git-handle.ts
@@ -160,13 +160,6 @@ export function applyGitEnv(
return handle.git.env(env);
}
-// Path-listing git commands must run with -z and be split here: without -z,
-// core.quotepath makes git C-quote non-ASCII filenames (e.g. "s\303\266in.md"),
-// and the escaped string matches nothing when reused as a pathspec.
-export function splitNulSeparatedPaths(out: string): string[] {
- return out.split('\0').filter((path) => path.length > 0);
-}
-
/**
* Create a SimpleGit instance rooted at `projectDir` with optional credential
* args and index file isolation. Env construction (and the reasons each var is
diff --git a/packages/server/src/git-paths.test.ts b/packages/server/src/git-paths.test.ts
new file mode 100644
index 000000000..0de7fe17a
--- /dev/null
+++ b/packages/server/src/git-paths.test.ts
@@ -0,0 +1,226 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { execSync } from 'node:child_process';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import simpleGit from 'simple-git';
+import {
+ listNameStatus,
+ listNames,
+ listPorcelainPaths,
+ listTreeLongEntries,
+ parseNameStatusZ,
+ parsePorcelainPaths,
+ parseTreeLongEntriesZ,
+ splitNulSeparatedPaths,
+} from './git-paths.ts';
+
+// A name git C-quotes in its default (non-`-z`) output — the failure the whole
+// module exists to prevent.
+const NON_ASCII = 'hyvää yötä.md';
+
+describe('splitNulSeparatedPaths', () => {
+ test('empty input yields no paths', () => {
+ expect(splitNulSeparatedPaths('')).toEqual([]);
+ });
+
+ test('drops the empty field after a trailing NUL', () => {
+ expect(splitNulSeparatedPaths('a.md\0b.md\0')).toEqual(['a.md', 'b.md']);
+ });
+
+ test('preserves non-ASCII bytes verbatim', () => {
+ expect(splitNulSeparatedPaths(`${NON_ASCII}\0`)).toEqual([NON_ASCII]);
+ });
+});
+
+describe('parsePorcelainPaths', () => {
+ test('empty input yields no paths', () => {
+ expect(parsePorcelainPaths('')).toEqual([]);
+ });
+
+ test('reads the path after the XY status prefix', () => {
+ expect(parsePorcelainPaths(' M a.md\0')).toEqual(['a.md']);
+ });
+
+ test('skips the origin record following a rename', () => {
+ // `R <new>\0<old>\0` — the origin path is a prefix-less following record.
+ expect(parsePorcelainPaths('R new.md\0old.md\0')).toEqual(['new.md']);
+ });
+
+ test('skips the origin record following a copy', () => {
+ expect(parsePorcelainPaths('C copy.md\0src.md\0')).toEqual(['copy.md']);
+ });
+
+ test('keeps non-ASCII names intact', () => {
+ expect(parsePorcelainPaths(` M ${NON_ASCII}\0`)).toEqual([NON_ASCII]);
+ });
+
+ test('ignores records shorter than a status prefix', () => {
+ expect(parsePorcelainPaths('\0 M a.md\0')).toEqual(['a.md']);
+ });
+});
+
+describe('parseNameStatusZ', () => {
+ test('non-rename row: from and to are the single path', () => {
+ expect(parseNameStatusZ('D\0a.md\0')).toEqual([{ status: 'D', from: 'a.md', to: 'a.md' }]);
+ });
+
+ test('rename row carries distinct from/to', () => {
+ expect(parseNameStatusZ('R100\0old.md\0new.md\0')).toEqual([
+ { status: 'R100', from: 'old.md', to: 'new.md' },
+ ]);
+ });
+
+ test('copy row carries distinct from/to', () => {
+ expect(parseNameStatusZ('C90\0src.md\0copy.md\0')).toEqual([
+ { status: 'C90', from: 'src.md', to: 'copy.md' },
+ ]);
+ });
+
+ test('reads a mixed rename + non-rename field stream', () => {
+ expect(parseNameStatusZ('R100\0old.md\0new.md\0M\0other.md\0')).toEqual([
+ { status: 'R100', from: 'old.md', to: 'new.md' },
+ { status: 'M', from: 'other.md', to: 'other.md' },
+ ]);
+ });
+
+ test('keeps non-ASCII from/to intact', () => {
+ expect(parseNameStatusZ(`R100\0${NON_ASCII}\0renamed ${NON_ASCII}\0`)).toEqual([
+ { status: 'R100', from: NON_ASCII, to: `renamed ${NON_ASCII}` },
+ ]);
+ });
+
+ test('empty input yields no rows', () => {
+ expect(parseNameStatusZ('')).toEqual([]);
+ });
+});
+
+describe('parseTreeLongEntriesZ', () => {
+ test('reads metadata and raw path from ls-tree --long output', () => {
+ const object = 'f'.repeat(40);
+ expect(parseTreeLongEntriesZ(`100644 blob ${object} 12\t${NON_ASCII}\0`)).toEqual([
+ { mode: '100644', type: 'blob', object, size: 12, path: NON_ASCII },
+ ]);
+ });
+
+ test('maps a dash size to 0 for non-blob entries', () => {
+ const object = 'a'.repeat(40);
+
+ expect(parseTreeLongEntriesZ(`040000 tree ${object} -\tdocs/\0`)).toEqual([
+ { mode: '040000', type: 'tree', object, size: 0, path: 'docs/' },
+ ]);
+ });
+
+ test('empty input yields no entries', () => {
+ expect(parseTreeLongEntriesZ('')).toEqual([]);
+ });
+});
+
+describe('path-listing wrappers (real git)', () => {
+ let projectDir: string;
+
+ function run(cmd: string): string {
+ return execSync(cmd, { cwd: projectDir, encoding: 'utf8' });
+ }
+
+ beforeEach(() => {
+ projectDir = mkdtempSync(join(tmpdir(), 'ok-git-paths-test-'));
+ run('git init -q -b main');
+ run('git config user.email "[email protected]"');
+ run('git config user.name "Test"');
+ run('git config commit.gpgsign false');
+ run('git config core.quotepath true');
+ });
+
+ afterEach(() => {
+ rmSync(projectDir, { recursive: true, force: true });
+ });
+
+ test('listNames returns the real UTF-8 path from ls-tree (no C-quoting)', async () => {
+ writeFileSync(join(projectDir, NON_ASCII), 'x\n');
+ run('git add -A');
+ run('git commit -q -m init');
+
+ const paths = await listNames(simpleGit(projectDir), ['ls-tree', '-r', '--name-only', 'HEAD']);
+
+ expect(paths).toEqual([NON_ASCII]);
+ });
+
+ test('listNames places -z before a -- pathspec separator', async () => {
+ writeFileSync(join(projectDir, NON_ASCII), 'x\n');
+ run('git add -A');
+ run('git commit -q -m init');
+
+ const paths = await listNames(simpleGit(projectDir), [
+ 'ls-tree',
+ '-r',
+ '--name-only',
+ 'HEAD',
+ '--',
+ NON_ASCII,
+ ]);
+
+ expect(paths).toEqual([NON_ASCII]);
+ });
+
+ test('listTreeLongEntries returns the real UTF-8 path and blob size', async () => {
+ writeFileSync(join(projectDir, NON_ASCII), 'x\n');
+ run('git add -A');
+ run('git commit -q -m init');
+
+ const entries = await listTreeLongEntries(simpleGit(projectDir), [
+ 'ls-tree',
+ '-r',
+ '--long',
+ 'HEAD',
+ ]);
+
+ expect(entries).toHaveLength(1);
+ expect(entries[0]?.path).toBe(NON_ASCII);
+ expect(entries[0]?.size).toBe(2);
+ });
+
+ test('listPorcelainPaths reports a non-ASCII working-tree change by its real name', async () => {
+ writeFileSync(join(projectDir, NON_ASCII), 'x\n');
+
+ const paths = await listPorcelainPaths(simpleGit(projectDir));
+
+ expect(paths).toContain(NON_ASCII);
+ });
+
+ test('listPorcelainPaths reports only the destination of a staged non-ASCII rename', async () => {
+ writeFileSync(join(projectDir, NON_ASCII), 'x\n');
+ run('git add -A');
+ run('git commit -q -m init');
+ const renamed = `renamed ${NON_ASCII}`;
+ run(`git mv "${NON_ASCII}" "${renamed}"`);
+
+ const paths = await listPorcelainPaths(simpleGit(projectDir));
+
+ expect(paths).toEqual([renamed]);
+ });
+
+ test('listNameStatus classifies a non-ASCII rename with real from/to paths', async () => {
+ writeFileSync(join(projectDir, NON_ASCII), 'x\n');
+ run('git add -A');
+ run('git commit -q -m init');
+ const renamed = `renamed ${NON_ASCII}`;
+ run(`git mv "${NON_ASCII}" "${renamed}"`);
+ run('git commit -q -m rename');
+
+ const rows = await listNameStatus(simpleGit(projectDir), [
+ 'diff-tree',
+ '-M',
+ '-r',
+ '--no-commit-id',
+ '--name-status',
+ 'HEAD^1',
+ 'HEAD',
+ ]);
+
+ expect(rows).toHaveLength(1);
+ expect(rows[0]?.status.startsWith('R')).toBe(true);
+ expect(rows[0]?.from).toBe(NON_ASCII);
+ expect(rows[0]?.to).toBe(renamed);
+ });
+});
diff --git a/packages/server/src/git-paths.ts b/packages/server/src/git-paths.ts
new file mode 100644
index 000000000..3a227ef13
--- /dev/null
+++ b/packages/server/src/git-paths.ts
@@ -0,0 +1,173 @@
+/**
+ * NUL-safe surface for every git command that lists or parses PATHS.
+ *
+ * Git C-quotes non-ASCII / unusual filenames in its default newline output
+ * (`core.quotepath`), e.g. `hyvää yötä.md` prints as
+ * `"hyv\303\244\303\244 y\303\266t\303\244.md"`. Reusing that escaped string as
+ * a pathspec matches nothing, which silently broke sync of non-ASCII-named
+ * files (deletions never committed, conflicts misrouted). Passing `-z` switches
+ * git to raw NUL-separated bytes; these wrappers own the `-z` flag AND the
+ * matching parser, so a new path-listing call site cannot reintroduce the
+ * quoting bug. Prefer `listNames` / `listPorcelainPaths` / `listNameStatus`
+ * over a bare `git.raw([...])` whenever the output is parsed as paths.
+ *
+ * Four record grammars are covered — each git command family emits its own, so
+ * a single "split on NUL" helper is not enough:
+ * - `--name-only -z` flat `path\0path\0…` -> splitNulSeparatedPaths
+ * - `status --porcelain -z` `XY path\0`, R/C add an origin record -> parsePorcelainPaths
+ * - `--name-status -z` `status\0path\0` or R/C `status\0from\0to\0` -> parseNameStatusZ
+ * - `ls-tree --long -z` `mode type object size\tpath\0` -> parseTreeLongEntriesZ
+ *
+ * No runtime dependency on `simple-git` (the SimpleGit import is type-only), so
+ * importing this module never triggers simple-git's load — callers that defer
+ * simple-git for test isolation can import these wrappers directly.
+ */
+import type { SimpleGit } from 'simple-git';
+
+/**
+ * Flat NUL-separated path list — the shape every `--name-only -z` command emits
+ * (`diff`, `diff-tree`, `diff-index`, `ls-tree`). The trailing empty field
+ * after the final NUL is dropped.
+ */
+export function splitNulSeparatedPaths(out: string): string[] {
+ return out.split('\0').filter((path) => path.length > 0);
+}
+
+/**
+ * `git status --porcelain -z`: NUL-separated `XY<space>PATH` records. A
+ * rename/copy emits the ORIGIN path as a separate following record with no
+ * status prefix, so skip that record — otherwise the old name reads as a
+ * distinct entry. Git detects renames only in the index (X) column under its
+ * defaults (worktree renames surface as delete+add), so the skip keys off
+ * `record[0]`; broaden to `record[1]` if a caller ever enables worktree-rename
+ * detection.
+ */
+export function parsePorcelainPaths(porcelain: string): string[] {
+ const paths: string[] = [];
+ const records = porcelain.split('\0');
+ for (let i = 0; i < records.length; i++) {
+ const record = records[i] ?? '';
+ if (record.length < 4) continue;
+ paths.push(record.slice(3));
+ if (record[0] === 'R' || record[0] === 'C') i++;
+ }
+ return paths;
+}
+
+/** One `--name-status` change. `from`/`to` are equal for non-rename statuses. */
+export interface NameStatusRow {
+ status: string;
+ from: string;
+ to: string;
+}
+
+/**
+ * `git diff[-tree] --name-status -z`: a flat NUL-separated FIELD stream, not one
+ * record per line. Each change is `status\0path\0`, except a rename/copy
+ * (`R`/`C`) which is `status\0from\0to\0`. Parsing the non-`-z` line/tab shape
+ * would both C-unescape-fail on non-ASCII paths and split a path on an embedded
+ * newline; consuming the field stream keeps every byte of the path intact.
+ */
+export function parseNameStatusZ(out: string): NameStatusRow[] {
+ const fields = out.split('\0');
+ const rows: NameStatusRow[] = [];
+ let i = 0;
+ while (i < fields.length) {
+ const status = fields[i] ?? '';
+ if (status === '') break; // trailing empty field after the final NUL
+ if (status[0] === 'R' || status[0] === 'C') {
+ rows.push({ status, from: fields[i + 1] ?? '', to: fields[i + 2] ?? '' });
+ i += 3;
+ } else {
+ const path = fields[i + 1] ?? '';
+ rows.push({ status, from: path, to: path });
+ i += 2;
+ }
+ }
+ return rows;
+}
+
+/** One `ls-tree --long -z` entry. */
+export interface TreeLongEntry {
+ mode: string;
+ type: string;
+ object: string;
+ size: number;
+ path: string;
+}
+
+/**
+ * `git ls-tree --long -z`: NUL-separated entries with a TAB between the
+ * metadata prefix and raw path. The size field can be `-` for non-blob entries;
+ * callers that need numeric sizes can treat that as 0.
+ */
+export function parseTreeLongEntriesZ(out: string): TreeLongEntry[] {
+ const entries: TreeLongEntry[] = [];
+ for (const record of out.split('\0')) {
+ if (!record) continue;
+ const tabIdx = record.indexOf('\t');
+ if (tabIdx < 0) continue;
+ const [mode = '', type = '', object = '', sizeRaw = '0'] = record
+ .slice(0, tabIdx)
+ .trim()
+ .split(/\s+/);
+ const size = Number(sizeRaw);
+ entries.push({
+ mode,
+ type,
+ object,
+ size: Number.isFinite(size) ? size : 0,
+ path: record.slice(tabIdx + 1),
+ });
+ }
+ return entries;
+}
+
+/**
+ * Insert `-z` immediately after the git subcommand so it precedes any positional
+ * revision / tree-ish / pathspec. `-z` is an option git accepts anywhere among
+ * the options, but appending it at the end would place it after a `--` pathspec,
+ * where git reads it as a path. `args[0]` MUST be the git subcommand.
+ */
+function rawZ(git: SimpleGit, args: string[]): Promise<string> {
+ const [subcommand = '', ...rest] = args;
+ return git.raw([subcommand, '-z', ...rest]);
+}
+
+/**
+ * Run a `--name-only` path-listing command (`diff`, `diff-tree`, `diff-index`,
+ * `ls-tree`) with `-z` and return the raw UTF-8 paths. Pass args WITHOUT `-z`.
+ */
+export async function listNames(git: SimpleGit, args: string[]): Promise<string[]> {
+ return splitNulSeparatedPaths(await rawZ(git, args));
+}
+
+/**
+ * Run `git status --porcelain` (plus any extra flags) with `-z` and return the
+ * paths, rename origin records skipped.
+ */
+export async function listPorcelainPaths(
+ git: SimpleGit,
+ args: string[] = ['status', '--porcelain'],
+): Promise<string[]> {
+ return parsePorcelainPaths(await rawZ(git, args));
+}
+
+/**
+ * Run a `--name-status` command (`diff`, `diff-tree`) with `-z` and return the
+ * parsed rows with real-UTF-8 `from`/`to` paths.
+ */
+export async function listNameStatus(git: SimpleGit, args: string[]): Promise<NameStatusRow[]> {
+ return parseNameStatusZ(await rawZ(git, args));
+}
+
+/**
+ * Run `git ls-tree --long` with `-z` and return parsed entries with real UTF-8
+ * paths and numeric blob sizes. Pass args WITHOUT `-z`.
+ */
+export async function listTreeLongEntries(
+ git: SimpleGit,
+ args: string[],
+): Promise<TreeLongEntry[]> {
+ return parseTreeLongEntriesZ(await rawZ(git, args));
+}
diff --git a/packages/server/src/shadow-repo.test.ts b/packages/server/src/shadow-repo.test.ts
index 6a94280c3..5ea94dd72 100644
--- a/packages/server/src/shadow-repo.test.ts
+++ b/packages/server/src/shadow-repo.test.ts
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test';
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { resolve } from 'node:path';
@@ -18,6 +18,7 @@ import {
GIT_UPSTREAM_WRITER,
type InMemoryCheckpointParams,
initShadowRepo,
+ listRescueCheckpoints,
type ParkableDoc,
parkBranch,
readParkedState,
@@ -916,6 +917,84 @@ describe('saveInMemoryCheckpoint (bridge-correctness SPEC §6 R7a)', () => {
expect(parsed.metadata.incomingDiskSha).toBe('abc123def456');
});
+ test('lists external-change-rescue checkpoints written by saveInMemoryCheckpoint', async () => {
+ const docName = 'hyvää yötä.md';
+ const contents = '# Rescued fast-path content\n';
+
+ const sha = await saveInMemoryCheckpoint(shadow, 'content/docs', {
+ kind: 'external-change-rescue',
+ docName,
+ contents,
+ label: 'Fast-path rescue',
+ metadata: { incomingDiskSha: 'abc123' },
+ });
+
+ const entries = await listRescueCheckpoints(shadow, 'main');
+
+ expect(entries).toHaveLength(1);
+ expect(entries[0]).toMatchObject({
+ docName,
+ size: Buffer.byteLength(contents, 'utf-8'),
+ sha,
+ label: 'Fast-path rescue',
+ incomingDiskSha: 'abc123',
+ });
+ expect(entries[0]?.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/);
+ });
+
+ test('lists legacy external-change-rescue checkpoints with non-ASCII docNames', async () => {
+ const docName = 'hyvää yötä.md';
+ const contents = '# Rescued legacy content\n';
+ const sg = shadowGit(shadow);
+ const tmpIndex = resolve(shadow.gitDir, 'index-legacy-rescue-test');
+ const tmpBlobFile = resolve(shadow.gitDir, 'tmp-legacy-rescue.md');
+
+ try {
+ writeFileSync(tmpBlobFile, contents);
+ const blobSha = (
+ await sg
+ .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
+ .raw('hash-object', '-w', tmpBlobFile)
+ ).trim();
+ await sg
+ .env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex })
+ .raw('update-index', '--add', '--cacheinfo', `100644,${blobSha},content/docs/${docName}`);
+ const treeSha = (
+ await sg.env({ GIT_DIR: shadow.gitDir, GIT_INDEX_FILE: tmpIndex }).raw('write-tree')
+ ).trim();
+ const body =
+ 'checkpoint: Legacy rescue\n\n' +
+ 'ok-checkpoint-v1: {"kind":"external-change-rescue","metadata":{"incomingDiskSha":"abc123"}}';
+ const sha = (
+ await sg
+ .env({
+ GIT_DIR: shadow.gitDir,
+ GIT_AUTHOR_NAME: 'openknowledge',
+ GIT_AUTHOR_EMAIL: '[email protected]',
+ GIT_COMMITTER_NAME: 'openknowledge',
+ GIT_COMMITTER_EMAIL: '[email protected]',
+ })
+ .raw('commit-tree', treeSha, '-m', body)
+ ).trim();
+ await sg.raw('update-ref', `refs/checkpoints/main/${sha}`, sha);
+
+ const entries = await listRescueCheckpoints(shadow, 'main');
+
+ expect(entries).toHaveLength(1);
+ expect(entries[0]).toMatchObject({
+ docName: 'hyvää yötä',
+ size: Buffer.byteLength(contents, 'utf-8'),
+ sha,
+ label: 'Legacy rescue',
+ incomingDiskSha: 'abc123',
+ });
+ expect(entries[0]?.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/);
+ } finally {
+ rmSync(tmpIndex, { force: true });
+ rmSync(tmpBlobFile, { force: true });
+ }
+ });
+
test('does NOT touch refs/wip/* — distinct from saveVersion', async () => {
// Create a WIP ref first via commitWip
const writer: WriterIdentity = {
diff --git a/packages/server/src/shadow-repo.ts b/packages/server/src/shadow-repo.ts
index a07885247..ac6dd13ee 100644
--- a/packages/server/src/shadow-repo.ts
+++ b/packages/server/src/shadow-repo.ts
@@ -43,6 +43,7 @@ import {
} from '@inkeep/open-knowledge-core/shadow-repo-layout';
import simpleGit from 'simple-git';
import { tracedMkdirSync, tracedRenameSync, tracedWriteFileSync } from './fs-traced.ts';
+import { listTreeLongEntries } from './git-paths.ts';
import { incrementShadowMigrationLegacyRefsDeleted } from './metrics.ts';
import { acquireLock, releaseLock } from './shadow-lock.ts';
import { withSpan } from './telemetry.ts';
@@ -951,15 +952,11 @@ export async function listRescueCheckpoints(
// populated both fields.
if (!docName) {
try {
- const tree = (await sg.raw('ls-tree', '-r', '--long', sha)).trim();
- const line = tree.split('\n')[0];
- if (line) {
- const cols = line.split(/\s+/);
- const pathIdx = 4;
- const sizeIdx = 3;
- if (size === 0) size = Number(cols[sizeIdx] ?? '0');
+ const entry = (await listTreeLongEntries(sg, ['ls-tree', '-r', '--long', sha]))[0];
+ if (entry) {
+ if (size === 0) size = entry.size;
docName =
- (cols[pathIdx] ?? '')
+ entry.path
.replace(/\.mdx?$/, '')
.split('/')
.slice(-1)[0] ?? '';
diff --git a/packages/server/src/share/target-status.ts b/packages/server/src/share/target-status.ts
index a8c71dafc..377f9fd9b 100644
--- a/packages/server/src/share/target-status.ts
+++ b/packages/server/src/share/target-status.ts
@@ -17,6 +17,7 @@ import { join } from 'node:path';
import type { ShareTargetStatusResponse } from '@inkeep/open-knowledge-core';
import { truncateError } from '../error-format.ts';
import { createGitInstance } from '../git-handle.ts';
+import { listNameStatus, type NameStatusRow } from '../git-paths.ts';
import { getLogger } from '../logger.ts';
/** Single source of truth for the handler tag used in logs + telemetry. */
@@ -25,34 +26,8 @@ export const SHARE_TARGET_STATUS_HANDLER_TAG = 'share-target-status';
/** Block timeout for the fetch — a hung fetch degrades to `unknown`. */
const DEFAULT_FETCH_TIMEOUT_MS = 15_000;
-interface DiffRow {
- status: string;
- from: string;
- to: string;
-}
-
-/**
- * Parse `git diff-tree --name-status` rows (tab-separated). Rename/copy rows
- * carry a score and two paths (`R100\told\tnew`); every other status carries
- * one path (`D\told`).
- */
-function parseNameStatus(output: string): DiffRow[] {
- const rows: DiffRow[] = [];
- for (const line of output.split('\n')) {
- if (line.trim() === '') continue;
- const parts = line.split('\t');
- const status = parts[0] ?? '';
- if ((status.startsWith('R') || status.startsWith('C')) && parts.length >= 3) {
- rows.push({ status, from: parts[1] ?? '', to: parts[2] ?? '' });
- } else {
- rows.push({ status, from: parts[1] ?? '', to: parts[1] ?? '' });
- }
- }
- return rows;
-}
-
/** Classify a doc removal: the row whose source is exactly the shared path. */
-function classifyDoc(rows: DiffRow[], gitPath: string): ShareTargetStatusResponse {
+function classifyDoc(rows: NameStatusRow[], gitPath: string): ShareTargetStatusResponse {
for (const row of rows) {
if (row.from !== gitPath) continue;
if (row.status.startsWith('R')) return { verdict: 'renamed', renamedTo: row.to };
@@ -69,7 +44,7 @@ function classifyDoc(rows: DiffRow[], gitPath: string): ShareTargetStatusRespons
* the common prefix across those rows; anything else (deletes, inconsistent
* destinations) is a deletion.
*/
-function classifyFolder(rows: DiffRow[], folderPath: string): ShareTargetStatusResponse {
+function classifyFolder(rows: NameStatusRow[], folderPath: string): ShareTargetStatusResponse {
const prefix = folderPath.endsWith('/') ? folderPath : `${folderPath}/`;
const newPrefixes = new Set<string>();
for (const row of rows) {
@@ -162,12 +137,12 @@ export async function computeShareTargetStatus(
// Diff the removing commit against its FIRST parent explicitly. A bare
// `diff-tree <commit>` on a merge commit emits combined-diff format (one
- // status column per parent), which `parseNameStatus` can't read — so a path
- // removed via a merge would misclassify. `<commit>^1 <commit>` forces the
- // standard single-status format and is equivalent to the bare form for
+ // status column per parent), which the name-status parse can't read — so a
+ // path removed via a merge would misclassify. `<commit>^1 <commit>` forces
+ // the standard single-status format and is equivalent to the bare form for
// ordinary single-parent commits. A removal only ever reachable through the
// second parent falls through to the honest `deleted` verdict.
- const nameStatus = await git.raw([
+ const rows = await listNameStatus(git, [
'diff-tree',
'-M',
'-r',
@@ -176,7 +151,6 @@ export async function computeShareTargetStatus(
`${removingCommit}^1`,
removingCommit,
]);
- const rows = parseNameStatus(nameStatus);
const classified =
kind === 'folder' ? classifyFolder(rows, gitPath) : classifyDoc(rows, gitPath);
diff --git a/packages/server/src/skill-restore.ts b/packages/server/src/skill-restore.ts
index 0c86a3e0e..ccedb7654 100644
--- a/packages/server/src/skill-restore.ts
+++ b/packages/server/src/skill-restore.ts
@@ -17,7 +17,7 @@
import { existsSync } from 'node:fs';
import { dirname, resolve, sep } from 'node:path';
import { tracedMkdirSync, tracedRmSync, tracedWriteFileSync } from './fs-traced.ts';
-import { splitNulSeparatedPaths } from './git-handle.ts';
+import { listNames } from './git-paths.ts';
import { type ShadowHandle, shadowGit } from './shadow-repo.ts';
/** Shadow-repo path of a project-scope skill's source dir for a content root. */
@@ -76,9 +76,9 @@ export async function restoreSkillVersion(opts: {
const shadowPath = skillShadowPath(contentRoot, name);
const sg = shadowGit(shadow);
- let fileList: string;
+ let files: string[];
try {
- fileList = await sg.raw('ls-tree', '-r', '--name-only', '-z', version, '--', shadowPath);
+ files = await listNames(sg, ['ls-tree', '-r', '--name-only', version, '--', shadowPath]);
} catch (e) {
// Distinguish "this commit/SHA genuinely isn't in the shadow repo" (a client
// 404) from a git I/O failure — git binary missing, repo corrupt, etc. (a
@@ -94,7 +94,6 @@ export async function restoreSkillVersion(opts: {
error: `Failed to read version ${version.slice(0, 8)}: ${msg}`,
};
}
- const files = splitNulSeparatedPaths(fileList);
if (files.length === 0) {
return {
ok: false,
diff --git a/packages/server/src/sync-engine.test.ts b/packages/server/src/sync-engine.test.ts
index 5e44a7c40..24e562853 100644
--- a/packages/server/src/sync-engine.test.ts
+++ b/packages/server/src/sync-engine.test.ts
@@ -27,6 +27,7 @@ import { join } from 'node:path';
import { LOCAL_DIR } from '@inkeep/open-knowledge-core';
import simpleGit from 'simple-git';
import { classifyGitError } from './error-classification.ts';
+import { listNames } from './git-paths.ts';
import type { DetectGhFn } from './github-permissions.ts';
import type { SyncState } from './sync-engine.ts';
import { SyncEngine } from './sync-engine.ts';
@@ -1344,9 +1345,7 @@ describe('SyncEngine push cycle with non-ASCII filenames', () => {
await engine.start();
await engine.trigger('push');
- const headPaths = (await git.raw(['ls-tree', '-r', '--name-only', '-z', 'HEAD']))
- .split('\0')
- .filter(Boolean);
+ const headPaths = await listNames(git, ['ls-tree', '-r', '--name-only', 'HEAD']);
expect(headPaths).toContain('README.md');
expect(headPaths).not.toContain(fileName);
diff --git a/packages/server/src/sync-engine.ts b/packages/server/src/sync-engine.ts
index 3f1e1a705..dd4ad23fb 100644
--- a/packages/server/src/sync-engine.ts
+++ b/packages/server/src/sync-engine.ts
@@ -30,14 +30,9 @@ import {
type UserFacingErrorCode,
} from './error-classification.ts';
import { createGhTokenSource, type GhTokenSource } from './gh-token-source.ts';
-import {
- applyGitEnv,
- createGitInstance,
- type GitHandle,
- splitNulSeparatedPaths,
- withParentLock,
-} from './git-handle.ts';
+import { applyGitEnv, createGitInstance, type GitHandle, withParentLock } from './git-handle.ts';
import { resolveGitIdentity } from './git-identity.ts';
+import { listNames } from './git-paths.ts';
import {
type CheckPushPermissionOptions,
type DetectGhFn,
@@ -490,8 +485,9 @@ export class SyncEngine {
// Merge still in progress — drop any tracked entries git considers resolved.
try {
const handle = this.gitHandle();
- const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
- const stillUnmerged = new Set(splitNulSeparatedPaths(out));
+ const stillUnmerged = new Set(
+ await listNames(handle.git, ['diff', '--name-only', '--diff-filter=U']),
+ );
const before = this.conflictCount;
for (const entry of this.conflictStore.list()) {
if (!stillUnmerged.has(entry.file)) {
@@ -1039,8 +1035,9 @@ export class SyncEngine {
} else {
try {
const handle = this.gitHandle();
- const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
- const stillUnmerged = new Set(splitNulSeparatedPaths(out));
+ const stillUnmerged = new Set(
+ await listNames(handle.git, ['diff', '--name-only', '--diff-filter=U']),
+ );
for (const entry of this.conflictStore.list()) {
if (!stillUnmerged.has(entry.file)) {
this.conflictStore.removeConflict(entry.file);
@@ -1445,15 +1442,13 @@ export class SyncEngine {
let changedProjectRelPaths: string[] = [];
let changedContentRelPaths: string[] = [];
try {
- const diffOut = await handle.git.raw([
+ const diffPaths = await listNames(handle.git, [
'diff-tree',
'--name-only',
- '-z',
'-r',
headSha,
newTreeSha,
]);
- const diffPaths = splitNulSeparatedPaths(diffOut);
if (diffPaths.length > 0) {
const contentFileByProjRel = new Map(
contentFiles.map((f) => [f.projectRelPath, f.contentRelPath]),
@@ -1669,15 +1664,13 @@ export class SyncEngine {
if (newTreeSha === headTreeSha) return null;
let changedProjectRelPaths: string[] = [];
try {
- const diffOut = await isoHandle.git.raw([
+ changedProjectRelPaths = await listNames(isoHandle.git, [
'diff-tree',
'--name-only',
- '-z',
'-r',
headSha,
newTreeSha,
]);
- changedProjectRelPaths = splitNulSeparatedPaths(diffOut);
} catch {
changedProjectRelPaths = contentFiles.map((f) => f.projectRelPath).concat(deleted);
}
@@ -1759,9 +1752,9 @@ export class SyncEngine {
// the merge would create the same path, which git surfaces at merge
// time with a specific error — we don't pre-pause for build artifacts,
// IDE state, or scratch notes.
- let dirtyOut = '';
+ let dirtyPaths: string[];
try {
- dirtyOut = await handle.git.raw(['diff-index', '--name-only', '-z', 'HEAD']);
+ dirtyPaths = await listNames(handle.git, ['diff-index', '--name-only', 'HEAD']);
} catch (err) {
// Fail-open is correct (git merge will surface real conflicts), but
// log so triage can spot a degraded pre-check (stale remote ref,
@@ -1769,22 +1762,24 @@ export class SyncEngine {
log.warn({ err, branch }, '[sync] diff-index failed — allowing merge attempt');
return { proceed: true, needsStashPop: false };
}
- const dirtyPaths = splitNulSeparatedPaths(dirtyOut);
if (dirtyPaths.length === 0) return { proceed: true, needsStashPop: false };
// Intersect with the set of paths the incoming merge actually touches.
// `diff --name-only HEAD..origin/<branch>` reports every path differing
- // between HEAD and the remote tip (renames decompose into delete+add
- // by default — a superset of paths that could conflict, which keeps
- // the intersection safe).
- let mergeOut = '';
+ // between HEAD and the remote tip. Git's rename detection reports only the
+ // rename DESTINATION, so a file dirty at its old name that the incoming
+ // merge renames is not caught here — the real protection is fail-open: the
+ // actual `git merge` still surfaces that conflict, this pre-check only
+ // avoids pausing on the common clean cases.
+ let mergePaths: Set<string>;
try {
- mergeOut = await handle.git.raw(['diff', '--name-only', '-z', `HEAD..origin/${branch}`]);
+ mergePaths = new Set(
+ await listNames(handle.git, ['diff', '--name-only', `HEAD..origin/${branch}`]),
+ );
} catch (err) {
log.warn({ err, branch }, '[sync] merge-path diff failed — allowing merge attempt');
return { proceed: true, needsStashPop: false };
}
- const mergePaths = new Set(splitNulSeparatedPaths(mergeOut));
const blocking = dirtyPaths.filter((p) => mergePaths.has(p));
if (blocking.length > 0) {
@@ -1871,8 +1866,8 @@ export class SyncEngine {
private async listHeadContentPaths(handle: GitHandle, headSha: string): Promise<Set<string>> {
const paths = new Set<string>();
try {
- const lsOut = await handle.git.raw(['ls-tree', '-r', '--name-only', '-z', headSha]);
- for (const projRelPath of splitNulSeparatedPaths(lsOut)) {
+ const headPaths = await listNames(handle.git, ['ls-tree', '-r', '--name-only', headSha]);
+ for (const projRelPath of headPaths) {
const absPath = join(this.projectDir, projRelPath);
const contentRelPath = toPosix(relative(this.contentDir, absPath));
if (!contentRelPath.startsWith('..') && !this.contentFilter.isExcluded(contentRelPath)) {
@@ -1938,8 +1933,7 @@ export class SyncEngine {
// produce a malformed commit.
let conflictedFiles: string[] = [];
try {
- const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
- conflictedFiles = splitNulSeparatedPaths(out);
+ conflictedFiles = await listNames(handle.git, ['diff', '--name-only', '--diff-filter=U']);
} catch (e) {
log.error(
{ err: e },
diff --git a/packages/server/src/upstream-attribution.integration.test.ts b/packages/server/src/upstream-attribution.integration.test.ts
index cb09502fd..2328b1b95 100644
--- a/packages/server/src/upstream-attribution.integration.test.ts
+++ b/packages/server/src/upstream-attribution.integration.test.ts
@@ -3,6 +3,7 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { resolve } from 'node:path';
import simpleGit, { type SimpleGit } from 'simple-git';
+import { __resetContributorsForTests as resetContributorsForTest } from './contributor-tracker.ts';
import { createServer, type ServerInstance } from './server-factory.ts';
import { initShadowRepo, type ShadowHandle, shadowGit } from './shadow-repo.ts';
import { getDocumentHistory } from './timeline-query.ts';
@@ -31,6 +32,12 @@ async function commitAs(name: string, email: string, file: string, body: string)
}
beforeEach(async () => {
+ // The pending-contributor accumulator is module-level and shared across every
+ // test file in the process. A stale `file-system` entry leaked by an earlier
+ // file would be drained into THIS test's shadow repo — and recordContributor's
+ // last-subject-wins merge can stamp it with this test's `reconcile:` subject,
+ // making the body assertions below match the wrong commit.
+ resetContributorsForTest();
dir = mkdtempSync(resolve(tmpdir(), 'ok-upstream-attr-int-'));
git = simpleGit(dir);
await git.init(['-b', 'main']);
@@ -77,18 +84,24 @@ test('merge while doc is open attributes the reconcile to the incoming author',
// color seed, asserted separately below.
const sg = shadowGit(shadow);
let reconcileLine = '';
+ let reconcileSha = '';
for (let i = 0; i < 40; i++) {
- const log = (await sg.raw('log', '--all', '--format=%an | %s')).trim();
- reconcileLine = log.split('\n').find((l) => l.startsWith('Ana Dev | reconcile:')) ?? '';
- if (reconcileLine) break;
+ const log = (await sg.raw('log', '--all', '--format=%H %an | %s')).trim();
+ const match = log.split('\n').find((l) => l.slice(41).startsWith('Ana Dev | reconcile:'));
+ if (match) {
+ reconcileSha = match.slice(0, 40);
+ reconcileLine = match.slice(41);
+ break;
+ }
await new Promise((r) => setTimeout(r, 250));
}
// The recovered git identity (real email) is carried on the ok-actor body line
- // as the color seed, not as the git-author email.
- const body = reconcileLine
- ? await sg.raw('log', '--all', '-1', '--format=%b', '--grep=reconcile: bugs')
- : '';
+ // as the color seed, not as the git-author email. Read the body by the SHA of
+ // the commit found above — a `--grep` here can race a sibling drain commit
+ // whose subject also says `reconcile: bugs` (e.g. the provisional file-system
+ // entry) and return the wrong body.
+ const body = reconcileSha ? await sg.raw('log', '-1', '--format=%b', reconcileSha) : '';
await conn.disconnect?.();