Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
chore(sync): mirror inkeep/open-knowledge#458 (#2412)
inkeep-internal-ci[bot]e95453cd4328
diff --git a/.changeset/fix-non-ascii-filename-sync.md b/.changeset/fix-non-ascii-filename-sync.mdnew file mode 100644index 000000000..faeacd59d--- /dev/null+++ b/.changeset/fix-non-ascii-filename-sync.md@@ -0,0 +1,11 @@+---+"@inkeep/open-knowledge": patch+---++Fix git sync mishandling files whose names contain non-ASCII characters (e.g. `ä`, `ö`, `å`, accents, CJK). Git quotes such names in its output (`"hyv\303\244\303\244 y\303\266t\303\244.md"`), and the sync engine reused that escaped string as a literal path, so:++1. Deleting a non-ASCII-named file was never committed — the deletion silently failed to sync on every cycle, forever, until the file was removed manually with `git rm`.+2. A merge conflict on a non-ASCII-named file was misclassified and could abort the merge and pause sync instead of surfacing the conflict for resolution.+3. Auto-save commit messages and the conflict list showed the escaped gibberish instead of the real filename.++Git path output is now read NUL-separated (`-z`), so filenames round-trip as real UTF-8 everywhere the server parses paths: the sync push/delete cycle, conflict detection and resolution, dirty-file overlap checks before branch switches, and skill version restore.diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.tsindex 6f9db3a26..a6445d93a 100644--- a/packages/server/src/boot.ts+++ b/packages/server/src/boot.ts@@ -43,6 +43,7 @@ import type { Config } from './config/schema.ts';import { ConflictStore } from './conflict-storage.ts';import { stripDocExtension } from './doc-extensions.ts';import { normalizeFsPath } from './fs-traced.ts';+import { splitNulSeparatedPaths } from './git-handle.ts';import {assertGitAvailable,type GitDetected,@@ -1086,15 +1087,8 @@ export async function restoreLifecycleFromConflictsJson(args: {return;}const pg = simpleGit({ baseDir: projectDir, timeout: { block: 5_000 } });- const out = (await pg.raw(['diff', '--name-only', '--diff-filter=U'])).trim();- stillUnmerged = new Set(- out- ? out- .split('\n')- .map((s) => s.trim())- .filter(Boolean)- : [],- );+ const out = await pg.raw(['diff', '--name-only', '--diff-filter=U', '-z']);+ stillUnmerged = new Set(splitNulSeparatedPaths(out));} catch (err) {// Probe failed — fall through and restore everything; the sync// engine's own reconcile on `start()` will mop up any stragglers.diff --git a/packages/server/src/conflict-storage.ts b/packages/server/src/conflict-storage.tsindex 82c3be5b4..1ec0eb14c 100644--- a/packages/server/src/conflict-storage.ts+++ b/packages/server/src/conflict-storage.ts@@ -255,11 +255,9 @@ export class ConflictStore {const detectedAt = new Date().toISOString();let reAdded = false;try {- const raw = await handle.git.raw(['diff', '--name-only', '--diff-filter=U']);- const unmerged = raw- .split('\n')- .map((s) => s.trim())- .filter(Boolean);+ const { splitNulSeparatedPaths } = await import('./git-handle.ts');+ const raw = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);+ const unmerged = splitNulSeparatedPaths(raw);for (const f of unmerged) {this.addConflict({ file: f, detectedAt });}diff --git a/packages/server/src/git-dirty.test.ts b/packages/server/src/git-dirty.test.tsindex c9546913e..177c4e21b 100644--- a/packages/server/src/git-dirty.test.ts+++ b/packages/server/src/git-dirty.test.ts@@ -198,6 +198,38 @@ describe('dirtyFilesOverlapWith', () => {expect(result.conflicts).toBe(true);expect(result.files).toEqual(['a.md']);});++ test('non-ASCII filename in the overlap is reported with its real name', async () => {+ write('hyvää yötä.md', 'base\n');+ commitAll('init');+ run('git checkout -q -b feature');+ write('hyvää yötä.md', 'feature edit\n');+ commitAll('feature edits non-ascii file');+ run('git checkout -q main');++ write('hyvää yötä.md', 'dirty\n');++ const result = await dirtyFilesOverlapWith(projectDir, 'feature');++ expect(result.conflicts).toBe(true);+ expect(result.files).toEqual(['hyvää yötä.md']);+ });++ test('staged rename reports the new path, not the original', async () => {+ write('old-name.md', 'base\n');+ commitAll('init');+ run('git checkout -q -b feature');+ write('new-name.md', 'feature version\n');+ commitAll('feature adds new-name');+ run('git checkout -q main');++ run('git mv old-name.md new-name.md');++ const result = await dirtyFilesOverlapWith(projectDir, 'feature');++ expect(result.conflicts).toBe(true);+ expect(result.files).toEqual(['new-name.md']);+ });});// Suppress unused-import warnings for lifecycle hooksdiff --git a/packages/server/src/git-dirty.ts b/packages/server/src/git-dirty.tsindex 8882a3ad0..014ad6236 100644--- a/packages/server/src/git-dirty.ts+++ b/packages/server/src/git-dirty.ts@@ -9,7 +9,7 @@* them.*/-import { createGitInstance } from './git-handle.ts';+import { createGitInstance, splitNulSeparatedPaths } from './git-handle.ts';export interface DirtyOverlapResult {conflicts: boolean;@@ -17,25 +17,19 @@ export interface DirtyOverlapResult {}function parsePorcelainPaths(porcelain: string): string[] {+ // -z porcelain: NUL-separated "XY PATH" records; a rename/copy emits the+ // original path as the following record, without a status prefix.const paths: string[] = [];- for (const line of porcelain.split('\n')) {- if (line.length < 4) continue;- // Porcelain v1 columns: XY <space> path (renames use 'orig -> new').- const rest = line.slice(3);- const renameIdx = rest.indexOf(' -> ');- const path = renameIdx >= 0 ? rest.slice(renameIdx + 4) : rest;- if (path.length > 0) paths.push(path);+ const records = porcelain.split('\0');+ for (let i = 0; i < records.length; i++) {+ const record = records[i] ?? '';+ if (record.length < 4) continue;+ paths.push(record.slice(3));+ if (record[0] === 'R' || record[0] === 'C') i++;}return paths;}-function parseDiffPaths(diffOutput: string): string[] {- return diffOutput- .split('\n')- .map((s) => s.trim())- .filter((s) => s.length > 0);-}-/*** Returns the intersection of dirty working-tree files and files changed by* switching from the current ref to `targetRef`.@@ -58,18 +52,18 @@ export async function dirtyFilesOverlapWith(const { git } = createGitInstance(cwd);const [porcelain, diff] = await Promise.all([- git.raw(['status', '--porcelain']),+ git.raw(['status', '--porcelain', '-z']),// Two-dot diff — all files differing between HEAD and targetRef, in either// direction. Three-dot (`HEAD...targetRef`) resolves to merge-base..targetRef// and misses files HEAD changed since divergence, which `git checkout` would// still restore to the target's view.- git.raw(['diff', '--name-only', `HEAD..${targetRef}`]),+ git.raw(['diff', '--name-only', '-z', `HEAD..${targetRef}`]),]);const dirty = new Set(parsePorcelainPaths(porcelain));if (dirty.size === 0) return { conflicts: false, files: [] };- const changed = parseDiffPaths(diff);+ const changed = splitNulSeparatedPaths(diff);if (changed.length === 0) return { conflicts: false, files: [] };const overlap = new Set<string>();diff --git a/packages/server/src/git-handle.ts b/packages/server/src/git-handle.tsindex b3ab8f9b7..8feacb222 100644--- a/packages/server/src/git-handle.ts+++ b/packages/server/src/git-handle.ts@@ -160,6 +160,13 @@ export function applyGitEnv(return handle.git.env(env);}+// Path-listing git commands must run with -z and be split here: without -z,+// core.quotepath makes git C-quote non-ASCII filenames (e.g. "s\303\266in.md"),+// and the escaped string matches nothing when reused as a pathspec.+export function splitNulSeparatedPaths(out: string): string[] {+ return out.split('\0').filter((path) => path.length > 0);+}+/*** Create a SimpleGit instance rooted at `projectDir` with optional credential* args and index file isolation. Env construction (and the reasons each var isdiff --git a/packages/server/src/skill-restore.ts b/packages/server/src/skill-restore.tsindex 25f2ce9d4..0c86a3e0e 100644--- a/packages/server/src/skill-restore.ts+++ b/packages/server/src/skill-restore.ts@@ -17,6 +17,7 @@import { existsSync } from 'node:fs';import { dirname, resolve, sep } from 'node:path';import { tracedMkdirSync, tracedRmSync, tracedWriteFileSync } from './fs-traced.ts';+import { splitNulSeparatedPaths } from './git-handle.ts';import { type ShadowHandle, shadowGit } from './shadow-repo.ts';/** Shadow-repo path of a project-scope skill's source dir for a content root. */@@ -77,7 +78,7 @@ export async function restoreSkillVersion(opts: {let fileList: string;try {- fileList = await sg.raw('ls-tree', '-r', '--name-only', version, '--', shadowPath);+ fileList = await sg.raw('ls-tree', '-r', '--name-only', '-z', version, '--', shadowPath);} catch (e) {// Distinguish "this commit/SHA genuinely isn't in the shadow repo" (a client// 404) from a git I/O failure — git binary missing, repo corrupt, etc. (a@@ -93,10 +94,7 @@ export async function restoreSkillVersion(opts: {error: `Failed to read version ${version.slice(0, 8)}: ${msg}`,};}- const files = fileList- .split('\n')- .map((s) => s.trim())- .filter(Boolean);+ const files = splitNulSeparatedPaths(fileList);if (files.length === 0) {return {ok: false,diff --git a/packages/server/src/sync-engine.test.ts b/packages/server/src/sync-engine.test.tsindex 2e8212874..1e04f5ffc 100644--- a/packages/server/src/sync-engine.test.ts+++ b/packages/server/src/sync-engine.test.ts@@ -829,6 +829,71 @@ describe('SyncEngine delete/modify dirty content conflicts', () => {});});+describe('SyncEngine non-ASCII filename conflicts', () => {+ const fileName = 'hyvää yötä.md';++ async function setupRemoteModifyLocalDeleteNonAscii(): Promise<void> {+ const bareDir = join(tmpDir, 'bare.git');+ mkdirSync(bareDir, { recursive: true });+ const bare = simpleGit(bareDir);+ await bare.init(true);++ const sisterDir = join(tmpDir, 'sister');+ mkdirSync(sisterDir, { recursive: true });+ const sister = simpleGit(sisterDir);+ await sister.init(['--initial-branch=main']);+ await sister.raw('config', 'user.name', 'Sister');+ await sister.raw('config', 'user.email', '[email protected]');+ writeFileSync(join(sisterDir, fileName), 'base\n', 'utf-8');+ await sister.add([fileName]);+ await sister.commit('base');+ await sister.addRemote('origin', bareDir);+ await sister.push(['--set-upstream', 'origin', 'main']);+ await bare.raw(['symbolic-ref', 'HEAD', 'refs/heads/main']);++ rmSync(projectDir, { recursive: true, force: true });+ await simpleGit(tmpDir).clone(bareDir, projectDir, ['--branch', 'main']);+ mkdirSync(okDir, { recursive: true });++ const project = simpleGit(projectDir);+ await project.raw('config', 'user.name', 'Project');+ await project.raw('config', 'user.email', '[email protected]');++ writeFileSync(join(sisterDir, fileName), 'remote edit\n', 'utf-8');+ await sister.add([fileName]);+ await sister.commit('remote modify');+ await sister.push('origin', 'main');++ rmSync(join(projectDir, fileName), { force: true });+ }++ test('surfaces a content conflict with the real UTF-8 path', async () => {+ await setupRemoteModifyLocalDeleteNonAscii();++ const engine = new SyncEngine({+ projectDir,+ contentDir: projectDir,+ contentFilter: stubContentFilter,+ syncEnabled: true,+ pullIntervalSeconds: 99999,+ pushIntervalSeconds: 99999,+ });+ try {+ await engine.start();+ await engine.trigger('sync');++ const status = engine.getStatus();+ expect(status.state).toBe('conflict');+ expect(status.conflictCount).toBe(1);+ expect(status.pausedReason).toBeUndefined();+ expect(engine.getConflicts().map((c) => c.file)).toEqual([fileName]);+ expect(existsSync(join(projectDir, '.git', 'MERGE_HEAD'))).toBe(true);+ } finally {+ await engine.destroy();+ }+ });+});+// ─── Status shape ─────────────────────────────────────────────────────────────describe('SyncEngine getStatus()', () => {@@ -1247,6 +1312,54 @@ describe('SyncEngine push cycle pushes existing commits when local is ahead of o});});+describe('SyncEngine push cycle with non-ASCII filenames', () => {+ const fileName = 'hyvää yötä.md';++ test('commits and pushes the deletion of a file with a non-ASCII name', async () => {+ const git = simpleGit(projectDir);+ await git.init(['--initial-branch=main']);+ await git.raw('config', 'user.name', 'Test');+ await git.raw('config', 'user.email', '[email protected]');+ writeFileSync(join(projectDir, 'README.md'), '# Test\n');+ writeFileSync(join(projectDir, fileName), 'sisältö\n');+ await git.add('.');+ await git.commit('add non-ascii file');++ const bareDir = join(tmpDir, 'bare.git');+ mkdirSync(bareDir, { recursive: true });+ await simpleGit(bareDir).init(true);+ await git.addRemote('origin', bareDir);+ await git.push(['--set-upstream', 'origin', 'main']);++ rmSync(join(projectDir, fileName));++ const engine = new SyncEngine({+ projectDir,+ contentDir: projectDir,+ contentFilter: stubContentFilter,+ syncEnabled: true,+ });+ try {+ await engine.start();+ await engine.trigger('push');++ const headPaths = (await git.raw(['ls-tree', '-r', '--name-only', '-z', 'HEAD']))+ .split('\0')+ .filter(Boolean);+ expect(headPaths).toContain('README.md');+ expect(headPaths).not.toContain(fileName);++ const subject = (await git.raw(['log', '--format=%s', '--max-count=1'])).trim();+ expect(subject).toContain(fileName);++ const remoteHead = (await git.revparse(['origin/main'])).trim();+ expect(remoteHead).toBe((await git.revparse(['HEAD'])).trim());+ } finally {+ await engine.destroy();+ }+ });+});+describe('SyncEngine per-operation error isolation', () => {// Regression: a single shared error field let a successful fetch clear a// failed push's error, so the sync popover flashed the push error for adiff --git a/packages/server/src/sync-engine.ts b/packages/server/src/sync-engine.tsindex bd07617d1..3f1e1a705 100644--- a/packages/server/src/sync-engine.ts+++ b/packages/server/src/sync-engine.ts@@ -30,7 +30,13 @@ import {type UserFacingErrorCode,} from './error-classification.ts';import { createGhTokenSource, type GhTokenSource } from './gh-token-source.ts';-import { applyGitEnv, createGitInstance, type GitHandle, withParentLock } from './git-handle.ts';+import {+ applyGitEnv,+ createGitInstance,+ type GitHandle,+ splitNulSeparatedPaths,+ withParentLock,+} from './git-handle.ts';import { resolveGitIdentity } from './git-identity.ts';import {type CheckPushPermissionOptions,@@ -484,15 +490,8 @@ export class SyncEngine {// Merge still in progress — drop any tracked entries git considers resolved.try {const handle = this.gitHandle();- const out = (await handle.git.raw(['diff', '--name-only', '--diff-filter=U'])).trim();- const stillUnmerged = new Set(- out- ? out- .split('\n')- .map((s) => s.trim())- .filter(Boolean)- : [],- );+ const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);+ const stillUnmerged = new Set(splitNulSeparatedPaths(out));const before = this.conflictCount;for (const entry of this.conflictStore.list()) {if (!stillUnmerged.has(entry.file)) {@@ -1040,15 +1039,8 @@ export class SyncEngine {} else {try {const handle = this.gitHandle();- const out = (await handle.git.raw(['diff', '--name-only', '--diff-filter=U'])).trim();- const stillUnmerged = new Set(- out- ? out- .split('\n')- .map((s) => s.trim())- .filter(Boolean)- : [],- );+ const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);+ const stillUnmerged = new Set(splitNulSeparatedPaths(out));for (const entry of this.conflictStore.list()) {if (!stillUnmerged.has(entry.file)) {this.conflictStore.removeConflict(entry.file);@@ -1453,16 +1445,20 @@ export class SyncEngine {let changedProjectRelPaths: string[] = [];let changedContentRelPaths: string[] = [];try {- const diffOut = (- await handle.git.raw(['diff-tree', '--name-only', '-r', headSha, newTreeSha])- ).trim();- if (diffOut) {+ const diffOut = await handle.git.raw([+ 'diff-tree',+ '--name-only',+ '-z',+ '-r',+ headSha,+ newTreeSha,+ ]);+ const diffPaths = splitNulSeparatedPaths(diffOut);+ if (diffPaths.length > 0) {const contentFileByProjRel = new Map(contentFiles.map((f) => [f.projectRelPath, f.contentRelPath]),);- for (const line of diffOut.split('\n')) {- const projRelPath = line.trim();- if (!projRelPath) continue;+ for (const projRelPath of diffPaths) {changedProjectRelPaths.push(projRelPath);const contentRelPath =contentFileByProjRel.get(projRelPath) ??@@ -1673,15 +1669,15 @@ export class SyncEngine {if (newTreeSha === headTreeSha) return null;let changedProjectRelPaths: string[] = [];try {- const diffOut = (- await isoHandle.git.raw(['diff-tree', '--name-only', '-r', headSha, newTreeSha])- ).trim();- changedProjectRelPaths = diffOut- ? diffOut- .split('\n')- .map((p) => p.trim())- .filter(Boolean)- : [];+ const diffOut = await isoHandle.git.raw([+ 'diff-tree',+ '--name-only',+ '-z',+ '-r',+ headSha,+ newTreeSha,+ ]);+ changedProjectRelPaths = splitNulSeparatedPaths(diffOut);} catch {changedProjectRelPaths = contentFiles.map((f) => f.projectRelPath).concat(deleted);}@@ -1765,7 +1761,7 @@ export class SyncEngine {// IDE state, or scratch notes.let dirtyOut = '';try {- dirtyOut = (await handle.git.raw(['diff-index', '--name-only', 'HEAD'])).trim();+ dirtyOut = await handle.git.raw(['diff-index', '--name-only', '-z', 'HEAD']);} catch (err) {// Fail-open is correct (git merge will surface real conflicts), but// log so triage can spot a degraded pre-check (stale remote ref,@@ -1773,11 +1769,7 @@ export class SyncEngine {log.warn({ err, branch }, '[sync] diff-index failed — allowing merge attempt');return { proceed: true, needsStashPop: false };}- if (!dirtyOut) return { proceed: true, needsStashPop: false };- const dirtyPaths = dirtyOut- .split('\n')- .map((p) => p.trim())- .filter(Boolean);+ const dirtyPaths = splitNulSeparatedPaths(dirtyOut);if (dirtyPaths.length === 0) return { proceed: true, needsStashPop: false };// Intersect with the set of paths the incoming merge actually touches.@@ -1787,17 +1779,12 @@ export class SyncEngine {// the intersection safe).let mergeOut = '';try {- mergeOut = (await handle.git.raw(['diff', '--name-only', `HEAD..origin/${branch}`])).trim();+ mergeOut = await handle.git.raw(['diff', '--name-only', '-z', `HEAD..origin/${branch}`]);} catch (err) {log.warn({ err, branch }, '[sync] merge-path diff failed — allowing merge attempt');return { proceed: true, needsStashPop: false };}- const mergePaths = new Set(- mergeOut- .split('\n')- .map((p) => p.trim())- .filter(Boolean),- );+ const mergePaths = new Set(splitNulSeparatedPaths(mergeOut));const blocking = dirtyPaths.filter((p) => mergePaths.has(p));if (blocking.length > 0) {@@ -1884,10 +1871,8 @@ export class SyncEngine {private async listHeadContentPaths(handle: GitHandle, headSha: string): Promise<Set<string>> {const paths = new Set<string>();try {- const lsOut = (await handle.git.raw(['ls-tree', '-r', '--name-only', headSha])).trim();- for (const line of lsOut ? lsOut.split('\n') : []) {- const projRelPath = line.trim();- if (!projRelPath) continue;+ const lsOut = await handle.git.raw(['ls-tree', '-r', '--name-only', '-z', headSha]);+ for (const projRelPath of splitNulSeparatedPaths(lsOut)) {const absPath = join(this.projectDir, projRelPath);const contentRelPath = toPosix(relative(this.contentDir, absPath));if (!contentRelPath.startsWith('..') && !this.contentFilter.isExcluded(contentRelPath)) {@@ -1953,13 +1938,8 @@ export class SyncEngine {// produce a malformed commit.let conflictedFiles: string[] = [];try {- const out = (await handle.git.raw(['diff', '--name-only', '--diff-filter=U'])).trim();- conflictedFiles = out- ? out- .split('\n')- .map((s) => s.trim())- .filter(Boolean)- : [];+ const out = await handle.git.raw(['diff', '--name-only', '--diff-filter=U', '-z']);+ conflictedFiles = splitNulSeparatedPaths(out);} catch (e) {log.error({ err: e },