Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

chore(sync): mirror inkeep/open-knowledge#458 (#2412)

inkeep-internal-ci[bot]e95453cd4328
diff --git a/.changeset/fix-non-ascii-filename-sync.md b/.changeset/fix-non-ascii-filename-sync.md
new file mode 100644
index 000000000..faeacd59d
--- /dev/null
+++ b/.changeset/fix-non-ascii-filename-sync.md
@@ -0,0 +1,11 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Fix git sync mishandling files whose names contain non-ASCII characters (e.g. `ä`, `ö`, `å`, accents, CJK). Git quotes such names in its output (`"hyv\303\244\303\244 y\303\266t\303\244.md"`), and the sync engine reused that escaped string as a literal path, so:
+
+1. Deleting a non-ASCII-named file was never committed — the deletion silently failed to sync on every cycle, forever, until the file was removed manually with `git rm`.
+2. A merge conflict on a non-ASCII-named file was misclassified and could abort the merge and pause sync instead of surfacing the conflict for resolution.
+3. Auto-save commit messages and the conflict list showed the escaped gibberish instead of the real filename.
+
+Git path output is now read NUL-separated (`-z`), so filenames round-trip as real UTF-8 everywhere the server parses paths: the sync push/delete cycle, conflict detection and resolution, dirty-file overlap checks before branch switches, and skill version restore.
diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.ts
index 6f9db3a26..a6445d93a 100644
--- a/packages/server/src/boot.ts
+++ b/packages/server/src/boot.ts
@@ -43,6 +43,7 @@ import type { Config } from './config/schema.ts';
import { ConflictStore } from './conflict-storage.ts';
import { stripDocExtension } from './doc-extensions.ts';
import { normalizeFsPath } from './fs-traced.ts';
+import { splitNulSeparatedPaths } from './git-handle.ts';
import {
assertGitAvailable,
type GitDetected,
@@ -1086,15 +1087,8 @@ export async function restoreLifecycleFromConflictsJson(args: {
return;
}
const pg = simpleGit({ baseDir: projectDir, timeout: { block: 5_000 } });
- const out = (await pg.raw(['diff', '--name-only', '--diff-filter=U'])).trim();
- stillUnmerged = new Set(
- out
- ? out
- .split('\n')
- .map((s) => s.trim())
- .filter(Boolean)
- : [],
- );
+ const out = await pg.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
+ stillUnmerged = new Set(splitNulSeparatedPaths(out));
} catch (err) {
// Probe failed — fall through and restore everything; the sync
// engine's own reconcile on `start()` will mop up any stragglers.
diff --git a/packages/server/src/conflict-storage.ts b/packages/server/src/conflict-storage.ts
index 82c3be5b4..1ec0eb14c 100644
--- a/packages/server/src/conflict-storage.ts
+++ b/packages/server/src/conflict-storage.ts
@@ -255,11 +255,9 @@ export class ConflictStore {
const detectedAt = new Date().toISOString();
let reAdded = false;
try {
- const raw = await handle.git.raw(['diff', '--name-only', '--diff-filter=U']);
- const unmerged = raw
- .split('\n')
- .map((s) => s.trim())
- .filter(Boolean);
+ const { splitNulSeparatedPaths } = await import('./git-handle.ts');
+ const raw = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
+ const unmerged = splitNulSeparatedPaths(raw);
for (const f of unmerged) {
this.addConflict({ file: f, detectedAt });
}
diff --git a/packages/server/src/git-dirty.test.ts b/packages/server/src/git-dirty.test.ts
index c9546913e..177c4e21b 100644
--- a/packages/server/src/git-dirty.test.ts
+++ b/packages/server/src/git-dirty.test.ts
@@ -198,6 +198,38 @@ describe('dirtyFilesOverlapWith', () => {
expect(result.conflicts).toBe(true);
expect(result.files).toEqual(['a.md']);
});
+
+ test('non-ASCII filename in the overlap is reported with its real name', async () => {
+ write('hyvää yötä.md', 'base\n');
+ commitAll('init');
+ run('git checkout -q -b feature');
+ write('hyvää yötä.md', 'feature edit\n');
+ commitAll('feature edits non-ascii file');
+ run('git checkout -q main');
+
+ write('hyvää yötä.md', 'dirty\n');
+
+ const result = await dirtyFilesOverlapWith(projectDir, 'feature');
+
+ expect(result.conflicts).toBe(true);
+ expect(result.files).toEqual(['hyvää yötä.md']);
+ });
+
+ test('staged rename reports the new path, not the original', async () => {
+ write('old-name.md', 'base\n');
+ commitAll('init');
+ run('git checkout -q -b feature');
+ write('new-name.md', 'feature version\n');
+ commitAll('feature adds new-name');
+ run('git checkout -q main');
+
+ run('git mv old-name.md new-name.md');
+
+ const result = await dirtyFilesOverlapWith(projectDir, 'feature');
+
+ expect(result.conflicts).toBe(true);
+ expect(result.files).toEqual(['new-name.md']);
+ });
});
// Suppress unused-import warnings for lifecycle hooks
diff --git a/packages/server/src/git-dirty.ts b/packages/server/src/git-dirty.ts
index 8882a3ad0..014ad6236 100644
--- a/packages/server/src/git-dirty.ts
+++ b/packages/server/src/git-dirty.ts
@@ -9,7 +9,7 @@
* them.
*/
-import { createGitInstance } from './git-handle.ts';
+import { createGitInstance, splitNulSeparatedPaths } from './git-handle.ts';
export interface DirtyOverlapResult {
conflicts: boolean;
@@ -17,25 +17,19 @@ export interface DirtyOverlapResult {
}
function parsePorcelainPaths(porcelain: string): string[] {
+ // -z porcelain: NUL-separated "XY PATH" records; a rename/copy emits the
+ // original path as the following record, without a status prefix.
const paths: string[] = [];
- for (const line of porcelain.split('\n')) {
- if (line.length < 4) continue;
- // Porcelain v1 columns: XY <space> path (renames use 'orig -> new').
- const rest = line.slice(3);
- const renameIdx = rest.indexOf(' -> ');
- const path = renameIdx >= 0 ? rest.slice(renameIdx + 4) : rest;
- if (path.length > 0) paths.push(path);
+ const records = porcelain.split('\0');
+ for (let i = 0; i < records.length; i++) {
+ const record = records[i] ?? '';
+ if (record.length < 4) continue;
+ paths.push(record.slice(3));
+ if (record[0] === 'R' || record[0] === 'C') i++;
}
return paths;
}
-function parseDiffPaths(diffOutput: string): string[] {
- return diffOutput
- .split('\n')
- .map((s) => s.trim())
- .filter((s) => s.length > 0);
-}
-
/**
* Returns the intersection of dirty working-tree files and files changed by
* switching from the current ref to `targetRef`.
@@ -58,18 +52,18 @@ export async function dirtyFilesOverlapWith(
const { git } = createGitInstance(cwd);
const [porcelain, diff] = await Promise.all([
- git.raw(['status', '--porcelain']),
+ git.raw(['status', '--porcelain', '-z']),
// Two-dot diff — all files differing between HEAD and targetRef, in either
// direction. Three-dot (`HEAD...targetRef`) resolves to merge-base..targetRef
// and misses files HEAD changed since divergence, which `git checkout` would
// still restore to the target's view.
- git.raw(['diff', '--name-only', `HEAD..${targetRef}`]),
+ git.raw(['diff', '--name-only', '-z', `HEAD..${targetRef}`]),
]);
const dirty = new Set(parsePorcelainPaths(porcelain));
if (dirty.size === 0) return { conflicts: false, files: [] };
- const changed = parseDiffPaths(diff);
+ const changed = splitNulSeparatedPaths(diff);
if (changed.length === 0) return { conflicts: false, files: [] };
const overlap = new Set<string>();
diff --git a/packages/server/src/git-handle.ts b/packages/server/src/git-handle.ts
index b3ab8f9b7..8feacb222 100644
--- a/packages/server/src/git-handle.ts
+++ b/packages/server/src/git-handle.ts
@@ -160,6 +160,13 @@ export function applyGitEnv(
return handle.git.env(env);
}
+// Path-listing git commands must run with -z and be split here: without -z,
+// core.quotepath makes git C-quote non-ASCII filenames (e.g. "s\303\266in.md"),
+// and the escaped string matches nothing when reused as a pathspec.
+export function splitNulSeparatedPaths(out: string): string[] {
+ return out.split('\0').filter((path) => path.length > 0);
+}
+
/**
* Create a SimpleGit instance rooted at `projectDir` with optional credential
* args and index file isolation. Env construction (and the reasons each var is
diff --git a/packages/server/src/skill-restore.ts b/packages/server/src/skill-restore.ts
index 25f2ce9d4..0c86a3e0e 100644
--- a/packages/server/src/skill-restore.ts
+++ b/packages/server/src/skill-restore.ts
@@ -17,6 +17,7 @@
import { existsSync } from 'node:fs';
import { dirname, resolve, sep } from 'node:path';
import { tracedMkdirSync, tracedRmSync, tracedWriteFileSync } from './fs-traced.ts';
+import { splitNulSeparatedPaths } from './git-handle.ts';
import { type ShadowHandle, shadowGit } from './shadow-repo.ts';
/** Shadow-repo path of a project-scope skill's source dir for a content root. */
@@ -77,7 +78,7 @@ export async function restoreSkillVersion(opts: {
let fileList: string;
try {
- fileList = await sg.raw('ls-tree', '-r', '--name-only', version, '--', shadowPath);
+ fileList = await sg.raw('ls-tree', '-r', '--name-only', '-z', version, '--', shadowPath);
} catch (e) {
// Distinguish "this commit/SHA genuinely isn't in the shadow repo" (a client
// 404) from a git I/O failure — git binary missing, repo corrupt, etc. (a
@@ -93,10 +94,7 @@ export async function restoreSkillVersion(opts: {
error: `Failed to read version ${version.slice(0, 8)}: ${msg}`,
};
}
- const files = fileList
- .split('\n')
- .map((s) => s.trim())
- .filter(Boolean);
+ const files = splitNulSeparatedPaths(fileList);
if (files.length === 0) {
return {
ok: false,
diff --git a/packages/server/src/sync-engine.test.ts b/packages/server/src/sync-engine.test.ts
index 2e8212874..1e04f5ffc 100644
--- a/packages/server/src/sync-engine.test.ts
+++ b/packages/server/src/sync-engine.test.ts
@@ -829,6 +829,71 @@ describe('SyncEngine delete/modify dirty content conflicts', () => {
});
});
+describe('SyncEngine non-ASCII filename conflicts', () => {
+ const fileName = 'hyvää yötä.md';
+
+ async function setupRemoteModifyLocalDeleteNonAscii(): Promise<void> {
+ const bareDir = join(tmpDir, 'bare.git');
+ mkdirSync(bareDir, { recursive: true });
+ const bare = simpleGit(bareDir);
+ await bare.init(true);
+
+ const sisterDir = join(tmpDir, 'sister');
+ mkdirSync(sisterDir, { recursive: true });
+ const sister = simpleGit(sisterDir);
+ await sister.init(['--initial-branch=main']);
+ await sister.raw('config', 'user.name', 'Sister');
+ await sister.raw('config', 'user.email', '[email protected]');
+ writeFileSync(join(sisterDir, fileName), 'base\n', 'utf-8');
+ await sister.add([fileName]);
+ await sister.commit('base');
+ await sister.addRemote('origin', bareDir);
+ await sister.push(['--set-upstream', 'origin', 'main']);
+ await bare.raw(['symbolic-ref', 'HEAD', 'refs/heads/main']);
+
+ rmSync(projectDir, { recursive: true, force: true });
+ await simpleGit(tmpDir).clone(bareDir, projectDir, ['--branch', 'main']);
+ mkdirSync(okDir, { recursive: true });
+
+ const project = simpleGit(projectDir);
+ await project.raw('config', 'user.name', 'Project');
+ await project.raw('config', 'user.email', '[email protected]');
+
+ writeFileSync(join(sisterDir, fileName), 'remote edit\n', 'utf-8');
+ await sister.add([fileName]);
+ await sister.commit('remote modify');
+ await sister.push('origin', 'main');
+
+ rmSync(join(projectDir, fileName), { force: true });
+ }
+
+ test('surfaces a content conflict with the real UTF-8 path', async () => {
+ await setupRemoteModifyLocalDeleteNonAscii();
+
+ const engine = new SyncEngine({
+ projectDir,
+ contentDir: projectDir,
+ contentFilter: stubContentFilter,
+ syncEnabled: true,
+ pullIntervalSeconds: 99999,
+ pushIntervalSeconds: 99999,
+ });
+ try {
+ await engine.start();
+ await engine.trigger('sync');
+
+ const status = engine.getStatus();
+ expect(status.state).toBe('conflict');
+ expect(status.conflictCount).toBe(1);
+ expect(status.pausedReason).toBeUndefined();
+ expect(engine.getConflicts().map((c) => c.file)).toEqual([fileName]);
+ expect(existsSync(join(projectDir, '.git', 'MERGE_HEAD'))).toBe(true);
+ } finally {
+ await engine.destroy();
+ }
+ });
+});
+
// ─── Status shape ─────────────────────────────────────────────────────────────
describe('SyncEngine getStatus()', () => {
@@ -1247,6 +1312,54 @@ describe('SyncEngine push cycle pushes existing commits when local is ahead of o
});
});
+describe('SyncEngine push cycle with non-ASCII filenames', () => {
+ const fileName = 'hyvää yötä.md';
+
+ test('commits and pushes the deletion of a file with a non-ASCII name', async () => {
+ const git = simpleGit(projectDir);
+ await git.init(['--initial-branch=main']);
+ await git.raw('config', 'user.name', 'Test');
+ await git.raw('config', 'user.email', '[email protected]');
+ writeFileSync(join(projectDir, 'README.md'), '# Test\n');
+ writeFileSync(join(projectDir, fileName), 'sisältö\n');
+ await git.add('.');
+ await git.commit('add non-ascii file');
+
+ const bareDir = join(tmpDir, 'bare.git');
+ mkdirSync(bareDir, { recursive: true });
+ await simpleGit(bareDir).init(true);
+ await git.addRemote('origin', bareDir);
+ await git.push(['--set-upstream', 'origin', 'main']);
+
+ rmSync(join(projectDir, fileName));
+
+ const engine = new SyncEngine({
+ projectDir,
+ contentDir: projectDir,
+ contentFilter: stubContentFilter,
+ syncEnabled: true,
+ });
+ try {
+ await engine.start();
+ await engine.trigger('push');
+
+ const headPaths = (await git.raw(['ls-tree', '-r', '--name-only', '-z', 'HEAD']))
+ .split('\0')
+ .filter(Boolean);
+ expect(headPaths).toContain('README.md');
+ expect(headPaths).not.toContain(fileName);
+
+ const subject = (await git.raw(['log', '--format=%s', '--max-count=1'])).trim();
+ expect(subject).toContain(fileName);
+
+ const remoteHead = (await git.revparse(['origin/main'])).trim();
+ expect(remoteHead).toBe((await git.revparse(['HEAD'])).trim());
+ } finally {
+ await engine.destroy();
+ }
+ });
+});
+
describe('SyncEngine per-operation error isolation', () => {
// Regression: a single shared error field let a successful fetch clear a
// failed push's error, so the sync popover flashed the push error for a
diff --git a/packages/server/src/sync-engine.ts b/packages/server/src/sync-engine.ts
index bd07617d1..3f1e1a705 100644
--- a/packages/server/src/sync-engine.ts
+++ b/packages/server/src/sync-engine.ts
@@ -30,7 +30,13 @@ import {
type UserFacingErrorCode,
} from './error-classification.ts';
import { createGhTokenSource, type GhTokenSource } from './gh-token-source.ts';
-import { applyGitEnv, createGitInstance, type GitHandle, withParentLock } from './git-handle.ts';
+import {
+ applyGitEnv,
+ createGitInstance,
+ type GitHandle,
+ splitNulSeparatedPaths,
+ withParentLock,
+} from './git-handle.ts';
import { resolveGitIdentity } from './git-identity.ts';
import {
type CheckPushPermissionOptions,
@@ -484,15 +490,8 @@ export class SyncEngine {
// Merge still in progress — drop any tracked entries git considers resolved.
try {
const handle = this.gitHandle();
- const out = (await handle.git.raw(['diff', '--name-only', '--diff-filter=U'])).trim();
- const stillUnmerged = new Set(
- out
- ? out
- .split('\n')
- .map((s) => s.trim())
- .filter(Boolean)
- : [],
- );
+ const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
+ const stillUnmerged = new Set(splitNulSeparatedPaths(out));
const before = this.conflictCount;
for (const entry of this.conflictStore.list()) {
if (!stillUnmerged.has(entry.file)) {
@@ -1040,15 +1039,8 @@ export class SyncEngine {
} else {
try {
const handle = this.gitHandle();
- const out = (await handle.git.raw(['diff', '--name-only', '--diff-filter=U'])).trim();
- const stillUnmerged = new Set(
- out
- ? out
- .split('\n')
- .map((s) => s.trim())
- .filter(Boolean)
- : [],
- );
+ const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
+ const stillUnmerged = new Set(splitNulSeparatedPaths(out));
for (const entry of this.conflictStore.list()) {
if (!stillUnmerged.has(entry.file)) {
this.conflictStore.removeConflict(entry.file);
@@ -1453,16 +1445,20 @@ export class SyncEngine {
let changedProjectRelPaths: string[] = [];
let changedContentRelPaths: string[] = [];
try {
- const diffOut = (
- await handle.git.raw(['diff-tree', '--name-only', '-r', headSha, newTreeSha])
- ).trim();
- if (diffOut) {
+ const diffOut = await handle.git.raw([
+ 'diff-tree',
+ '--name-only',
+ '-z',
+ '-r',
+ headSha,
+ newTreeSha,
+ ]);
+ const diffPaths = splitNulSeparatedPaths(diffOut);
+ if (diffPaths.length > 0) {
const contentFileByProjRel = new Map(
contentFiles.map((f) => [f.projectRelPath, f.contentRelPath]),
);
- for (const line of diffOut.split('\n')) {
- const projRelPath = line.trim();
- if (!projRelPath) continue;
+ for (const projRelPath of diffPaths) {
changedProjectRelPaths.push(projRelPath);
const contentRelPath =
contentFileByProjRel.get(projRelPath) ??
@@ -1673,15 +1669,15 @@ export class SyncEngine {
if (newTreeSha === headTreeSha) return null;
let changedProjectRelPaths: string[] = [];
try {
- const diffOut = (
- await isoHandle.git.raw(['diff-tree', '--name-only', '-r', headSha, newTreeSha])
- ).trim();
- changedProjectRelPaths = diffOut
- ? diffOut
- .split('\n')
- .map((p) => p.trim())
- .filter(Boolean)
- : [];
+ const diffOut = await isoHandle.git.raw([
+ 'diff-tree',
+ '--name-only',
+ '-z',
+ '-r',
+ headSha,
+ newTreeSha,
+ ]);
+ changedProjectRelPaths = splitNulSeparatedPaths(diffOut);
} catch {
changedProjectRelPaths = contentFiles.map((f) => f.projectRelPath).concat(deleted);
}
@@ -1765,7 +1761,7 @@ export class SyncEngine {
// IDE state, or scratch notes.
let dirtyOut = '';
try {
- dirtyOut = (await handle.git.raw(['diff-index', '--name-only', 'HEAD'])).trim();
+ dirtyOut = await handle.git.raw(['diff-index', '--name-only', '-z', 'HEAD']);
} catch (err) {
// Fail-open is correct (git merge will surface real conflicts), but
// log so triage can spot a degraded pre-check (stale remote ref,
@@ -1773,11 +1769,7 @@ export class SyncEngine {
log.warn({ err, branch }, '[sync] diff-index failed — allowing merge attempt');
return { proceed: true, needsStashPop: false };
}
- if (!dirtyOut) return { proceed: true, needsStashPop: false };
- const dirtyPaths = dirtyOut
- .split('\n')
- .map((p) => p.trim())
- .filter(Boolean);
+ const dirtyPaths = splitNulSeparatedPaths(dirtyOut);
if (dirtyPaths.length === 0) return { proceed: true, needsStashPop: false };
// Intersect with the set of paths the incoming merge actually touches.
@@ -1787,17 +1779,12 @@ export class SyncEngine {
// the intersection safe).
let mergeOut = '';
try {
- mergeOut = (await handle.git.raw(['diff', '--name-only', `HEAD..origin/${branch}`])).trim();
+ mergeOut = await handle.git.raw(['diff', '--name-only', '-z', `HEAD..origin/${branch}`]);
} catch (err) {
log.warn({ err, branch }, '[sync] merge-path diff failed — allowing merge attempt');
return { proceed: true, needsStashPop: false };
}
- const mergePaths = new Set(
- mergeOut
- .split('\n')
- .map((p) => p.trim())
- .filter(Boolean),
- );
+ const mergePaths = new Set(splitNulSeparatedPaths(mergeOut));
const blocking = dirtyPaths.filter((p) => mergePaths.has(p));
if (blocking.length > 0) {
@@ -1884,10 +1871,8 @@ export class SyncEngine {
private async listHeadContentPaths(handle: GitHandle, headSha: string): Promise<Set<string>> {
const paths = new Set<string>();
try {
- const lsOut = (await handle.git.raw(['ls-tree', '-r', '--name-only', headSha])).trim();
- for (const line of lsOut ? lsOut.split('\n') : []) {
- const projRelPath = line.trim();
- if (!projRelPath) continue;
+ const lsOut = await handle.git.raw(['ls-tree', '-r', '--name-only', '-z', headSha]);
+ for (const projRelPath of splitNulSeparatedPaths(lsOut)) {
const absPath = join(this.projectDir, projRelPath);
const contentRelPath = toPosix(relative(this.contentDir, absPath));
if (!contentRelPath.startsWith('..') && !this.contentFilter.isExcluded(contentRelPath)) {
@@ -1953,13 +1938,8 @@ export class SyncEngine {
// produce a malformed commit.
let conflictedFiles: string[] = [];
try {
- const out = (await handle.git.raw(['diff', '--name-only', '--diff-filter=U'])).trim();
- conflictedFiles = out
- ? out
- .split('\n')
- .map((s) => s.trim())
- .filter(Boolean)
- : [];
+ const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);
+ conflictedFiles = splitNulSeparatedPaths(out);
} catch (e) {
log.error(
{ err: e },