Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
Keep Unicode shadow histories and edit backups apart (PRD-9407) (#6411)
Andrew Mikofalvy0aeb2a1370fe
diff --git a/.changeset/injective-storage-keys.md b/.changeset/injective-storage-keys.mdnew file mode 100644index 000000000..2fe698be1--- /dev/null+++ b/.changeset/injective-storage-keys.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Projects in one git repository whose folder names differ only in non-Latin letters, spaces or punctuation now keep separate version histories, and such skill files keep separate discarded-edit backups. A history two projects already shared stays with the project that created it; the other starts fresh.diff --git a/packages/core/src/git-repository.ts b/packages/core/src/git-repository.tsindex 1fb1e33bf..75edc89f4 100644--- a/packages/core/src/git-repository.ts+++ b/packages/core/src/git-repository.ts@@ -281,7 +281,7 @@ function unquote(value: string): string {return trimmed;}-function isMissing(cause: unknown): boolean {+export function isMissing(cause: unknown): boolean {const code = (cause as NodeJS.ErrnoException | undefined)?.code;return code === 'ENOENT' || code === 'ENOTDIR';}diff --git a/packages/core/src/shadow-repo-layout.test.ts b/packages/core/src/shadow-repo-layout.test.tsindex 10340bd44..e9994bb96 100644--- a/packages/core/src/shadow-repo-layout.test.ts+++ b/packages/core/src/shadow-repo-layout.test.ts@@ -1,7 +1,8 @@-import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';+import { execFileSync } from 'node:child_process';+import { chmodSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { mkdtemp, rm } from 'node:fs/promises';import { tmpdir } from 'node:os';-import { resolve } from 'node:path';+import { basename, dirname, resolve } from 'node:path';import { afterEach, beforeEach, describe, expect, test } from 'vitest';import { runningAsRoot } from '../../../test-support/capabilities.test-helper.ts';import {@@ -559,6 +560,129 @@ describe('resolveShadowDir', () => {expect(resolveShadowDir(project)).toBe(resolve(project, '.git/ok'));});});+describe('resolveShadowDir sub-path keys are injective', () => {+ function legacyShadowName(rel: string): string {+ const flat = rel.split('/').join('-');+ return `ok-${flat.replace(/[^A-Za-z0-9._-]/g, '_').replace(/^\.+/, '_')}`;+ }++ function fakeShadow(dir: string, worktree: string): void {+ mkdirSync(dir, { recursive: true });+ writeFileSync(resolve(dir, 'HEAD'), 'ref: refs/heads/main\n');+ writeFileSync(+ resolve(dir, 'config'),+ `[core]\n\trepositoryformatversion = 0\n\tworktree = ${worktree}\n[user]\n\tname = openknowledge\n`,+ );+ }++ function makeRepo(...subs: string[]): string {+ const repo = resolve(tmp, 'repo');+ mkdirSync(resolve(repo, '.git'), { recursive: true });+ for (const sub of subs) mkdirSync(resolve(repo, sub), { recursive: true });+ return repo;+ }++ test('subfolders whose names differ only in non-Latin letters get distinct shadow dirs', () => {+ const repo = makeRepo('ノート', '日記帳');+ const a = resolveShadowDir(resolve(repo, 'ノート'));+ const b = resolveShadowDir(resolve(repo, '日記帳'));+ expect(a).not.toBe(b);+ for (const dir of [a, b]) {+ expect(dirname(dir)).toBe(resolve(repo, '.git'));+ expect(basename(dir)).toMatch(/^ok-[A-Za-z0-9._-]{1,64}$/);+ }+ });++ test('names that differ only in punctuation or spaces get distinct shadow dirs', () => {+ const repo = makeRepo('My Notes', 'My_Notes', 'My#Notes');+ const dirs = ['My Notes', 'My_Notes', 'My#Notes'].map((s) =>+ resolveShadowDir(resolve(repo, s)),+ );+ expect(new Set(dirs).size).toBe(3);+ });++ test('ASCII-safe sub-paths keep the exact key they had before', () => {+ const repo = makeRepo('my-notes_v2.0', 'public/open-knowledge');+ expect(resolveShadowDir(resolve(repo, 'my-notes_v2.0'))).toBe(+ resolve(repo, '.git/ok-my-notes_v2.0'),+ );+ expect(resolveShadowDir(resolve(repo, 'public/open-knowledge'))).toBe(+ resolve(repo, '.git/ok-public-open-knowledge'),+ );+ });++ test('a lossy shadow dir created by an earlier version stays with the project that owns it', () => {+ const repo = makeRepo('ノート', '日記帳');+ const legacy = resolve(repo, '.git', legacyShadowName('ノート'));+ expect(legacy).toBe(resolve(repo, '.git', legacyShadowName('日記帳')));+ fakeShadow(legacy, resolve(repo, 'ノート'));+ expect(resolveShadowDir(resolve(repo, 'ノート'))).toBe(legacy);+ expect(getShadowRepoPath(resolve(repo, 'ノート'))).toBe(legacy);+ expect(resolveShadowDir(resolve(repo, '日記帳'))).not.toBe(legacy);+ });++ test('legacy ownership survives the repository moving to another path', () => {+ const repo = makeRepo('ノート');+ const legacy = resolve(repo, '.git', legacyShadowName('ノート'));+ fakeShadow(legacy, resolve(tmp, 'old-location', 'repo', 'ノート'));+ expect(resolveShadowDir(resolve(repo, 'ノート'))).toBe(legacy);+ });++ test('legacy ownership reads a core.worktree value that git itself wrote', () => {+ const sub = 'メモ #1; a b';+ const repo = makeRepo(sub, 'メモ');+ const legacy = resolve(repo, '.git', legacyShadowName(sub));+ execFileSync('git', ['init', '--bare', '--quiet', legacy]);+ execFileSync('git', ['config', 'core.worktree', resolve(repo, sub)], {+ env: { ...process.env, GIT_DIR: legacy },+ });+ expect(resolveShadowDir(resolve(repo, sub))).toBe(legacy);+ expect(resolveShadowDir(resolve(repo, 'メモ'))).not.toBe(legacy);+ });++ test('a lossy legacy dir with no recorded owner is kept rather than abandoned', () => {+ const repo = makeRepo('ノート');+ const legacy = resolve(repo, '.git', legacyShadowName('ノート'));+ mkdirSync(legacy, { recursive: true });+ writeFileSync(resolve(legacy, 'HEAD'), 'ref: refs/heads/main\n');+ writeFileSync(resolve(legacy, 'config'), '[core]\n\trepositoryformatversion = 0\n');+ expect(resolveShadowDir(resolve(repo, 'ノート'))).toBe(legacy);+ });++ test('an unreadable legacy config keeps the legacy dir rather than moving to a new one', () => {+ const repo = makeRepo('ノート');+ const legacy = resolve(repo, '.git', legacyShadowName('ノート'));+ mkdirSync(resolve(legacy, 'config'), { recursive: true });+ expect(resolveShadowDir(resolve(repo, 'ノート'))).toBe(legacy);+ });++ test('a separator and a hyphen no longer share a dir once the hyphenated one owns it', () => {+ const repo = makeRepo('a-b', 'a/b');+ const shared = resolve(repo, '.git/ok-a-b');+ fakeShadow(shared, resolve(repo, 'a-b'));+ expect(resolveShadowDir(resolve(repo, 'a-b'))).toBe(shared);+ const nested = resolveShadowDir(resolve(repo, 'a/b'));+ expect(nested).not.toBe(shared);+ expect(basename(nested)).toMatch(/^ok-[A-Za-z0-9._-]{1,64}$/);+ });++ test('a project that moved off a foreign-owned dir keeps its own dir after the owner leaves', () => {+ const repo = makeRepo('a-b', 'a/b');+ const shared = resolve(repo, '.git/ok-a-b');+ fakeShadow(shared, resolve(repo, 'a-b'));+ const own = resolveShadowDir(resolve(repo, 'a/b'));+ fakeShadow(own, resolve(repo, 'a/b'));+ rmSync(shared, { recursive: true, force: true });+ expect(resolveShadowDir(resolve(repo, 'a/b'))).toBe(own);+ });++ test('NFC and NFD spellings of one name resolve to the same shadow dir', () => {+ const repo = makeRepo('café');+ const nfc = resolveShadowDir(resolve(repo, 'café'.normalize('NFC')));+ const nfd = resolveShadowDir(resolve(repo, 'café'.normalize('NFD')));+ expect(nfd).toBe(nfc);+ });+});describe('getShadowRepoPath', () => {test('returns null when no shadow repo exists', () => {diff --git a/packages/core/src/shadow-repo-layout.ts b/packages/core/src/shadow-repo-layout.tsindex 496248a3d..068b69cd6 100644--- a/packages/core/src/shadow-repo-layout.ts+++ b/packages/core/src/shadow-repo-layout.ts@@ -1,7 +1,8 @@+import { createHash } from 'node:crypto';import { existsSync, readFileSync } from 'node:fs';import { dirname, join, resolve, sep } from 'node:path';import { fnv1aDigest } from './bridge/hash-util.ts';-import { discoverGitRepository } from './git-repository.ts';+import { discoverGitRepository, isMissing } from './git-repository.ts';/** Writer-ID taxonomy (precedent #25). */export type WriterClassification =@@ -77,12 +78,12 @@ export function resolveShadowDir(projectRoot: string): string {const result = resolveGitDirDetailed(projectRoot);switch (result.kind) {case 'directory':- return resolve(result.path, shadowSubdirName(result.projectSubPath));+ return resolve(result.path, shadowSubdirName(result.path, result.projectSubPath));case 'linked':if (!existsSync(result.path)) {throw new MalformedGitPointerError(result.gitPath, result.path);}- return resolve(result.path, shadowSubdirName(result.projectSubPath));+ return resolve(result.path, shadowSubdirName(result.path, result.projectSubPath));case 'malformed-pointer':throw new MalformedGitPointerError(result.gitPath, result.target, { cause: result.cause });case 'inaccessible':@@ -92,18 +93,121 @@ export function resolveShadowDir(projectRoot: string): string {}}-function shadowSubdirName(projectSubPath: string): string {+function shadowSubdirName(gitDir: string, projectSubPath: string): string {if (projectSubPath === '') return 'ok';- return `ok-${slugifyShadowSubPath(projectSubPath)}`;+ const hashed = `ok-${hashedShadowSlug(projectSubPath)}`;+ if (existsSync(resolve(gitDir, hashed))) return hashed;+ const legacy = `ok-${legacyShadowSlug(projectSubPath)}`;+ switch (shadowOwner(resolve(gitDir, legacy), projectSubPath)) {+ case 'ours':+ case 'unrecorded':+ return legacy;+ case 'theirs':+ return hashed;+ case 'absent':+ return SAFE_SHADOW_SLUG_RE.test(flattenSubPath(projectSubPath)) ? legacy : hashed;+ }+}++const SAFE_SHADOW_SLUG_RE = /^[A-Za-z0-9_-][A-Za-z0-9._-]*$/;+const SHADOW_SLUG_MAX = 64;+const SHADOW_DIGEST_LEN = 12;++function flattenSubPath(rel: string): string {+ return rel.split(sep).join('-').replace(/\/+/g, '-');+}++function sanitizeShadowSlug(flat: string): string {+ return flat.replace(/[^A-Za-z0-9._-]/g, '_').replace(/^\.+/, '_');}-function slugifyShadowSubPath(rel: string): string {- const flat = rel.split(sep).join('-').replace(/\/+/g, '-');- const sanitized = flat.replace(/[^A-Za-z0-9._-]/g, '_').replace(/^\.+/, '_');- const MAX = 64;- if (sanitized.length <= MAX) return sanitized || 'sub';+function legacyShadowSlug(rel: string): string {+ const sanitized = sanitizeShadowSlug(flattenSubPath(rel));+ if (sanitized.length <= SHADOW_SLUG_MAX) return sanitized || 'sub';const hash = djb2(rel).toString(16).padStart(8, '0');- return `${sanitized.slice(0, MAX - 9)}-${hash}`;+ return `${sanitized.slice(0, SHADOW_SLUG_MAX - 9)}-${hash}`;+}++function hashedShadowSlug(rel: string): string {+ const canonical = canonicalSubPath(rel);+ const digest = createHash('sha256').update(canonical).digest('hex').slice(0, SHADOW_DIGEST_LEN);+ const readable = sanitizeShadowSlug(canonical.split('/').join('-')).slice(+ 0,+ SHADOW_SLUG_MAX - SHADOW_DIGEST_LEN - 1,+ );+ return `${readable || 'sub'}-${digest}`;+}++function canonicalSubPath(rel: string): string {+ return rel.normalize('NFC').split(sep).join('/').replace(/\/+/g, '/');+}++const CASE_INSENSITIVE_PATHS = process.platform === 'darwin' || process.platform === 'win32';++function comparablePath(p: string): string {+ const s = p.normalize('NFC').replace(/\\/g, '/').replace(/\/+$/, '');+ return CASE_INSENSITIVE_PATHS ? s.toLowerCase() : s;+}++function shadowOwner(+ shadowDir: string,+ projectSubPath: string,+): 'ours' | 'theirs' | 'unrecorded' | 'absent' {+ let config: string;+ try {+ config = readFileSync(resolve(shadowDir, 'config'), 'utf-8');+ } catch (cause) {+ return isMissing(cause) ? 'absent' : 'unrecorded';+ }+ const worktree = readCoreWorktree(config);+ if (worktree === null || worktree === '') return 'unrecorded';+ const owner = comparablePath(worktree);+ const sub = comparablePath(canonicalSubPath(projectSubPath));+ return owner === sub || owner.endsWith(`/${sub}`) ? 'ours' : 'theirs';+}++function readCoreWorktree(config: string): string | null {+ let inCore = false;+ let worktree: string | null = null;+ for (const rawLine of config.split(/\r?\n/)) {+ const line = rawLine.trim();+ if (line.startsWith('[')) {+ inCore = /^\[\s*core\s*\]/i.test(line);+ continue;+ }+ if (!inCore) continue;+ const match = /^worktree\s*=(.*)$/i.exec(line);+ if (match) worktree = parseGitConfigValue(match[1] ?? '');+ }+ return worktree;+}++function parseGitConfigValue(raw: string): string {+ let out = '';+ let pendingSpace = '';+ let quoted = false;+ for (let i = 0; i < raw.length; i++) {+ const c = raw[i];+ if (c === '\\') {+ const next = raw[++i];+ out +=+ pendingSpace ++ (next === 'n' ? '\n' : next === 't' ? '\t' : next === 'b' ? '\b' : (next ?? ''));+ pendingSpace = '';+ } else if (c === '"') {+ quoted = !quoted;+ out += pendingSpace;+ pendingSpace = '';+ } else if (!quoted && (c === ';' || c === '#')) {+ break;+ } else if (!quoted && (c === ' ' || c === '\t')) {+ if (out.length > 0) pendingSpace += c;+ } else {+ out += pendingSpace + c;+ pendingSpace = '';+ }+ }+ return out;}function djb2(s: string): number {diff --git a/packages/server/src/managed-artifact-persistence.test.ts b/packages/server/src/managed-artifact-persistence.test.tsindex 1329258e1..01bf6e178 100644--- a/packages/server/src/managed-artifact-persistence.test.ts+++ b/packages/server/src/managed-artifact-persistence.test.ts@@ -328,6 +328,69 @@ describe('store/load round-trip', () => {});});+describe('discarded-edit backups', () => {+ beforeEach(adoptClaudeHost);++ async function reconcileLocalEdit(docName: string, localEdit: string): Promise<void> {+ const ctx = makeCtx();+ const path = managedArtifactAbsPath(docName, ctx);+ mkdirSync(dirname(path), { recursive: true });+ const doc = new Y.Doc();+ doc.transact(() => doc.getText('source').insert(0, 'base\n'), 'agent');+ await storeManagedArtifactDoc(doc, docName, 'agent', ctx);+ writeFileSync(path, 'other writer\n', 'utf-8');+ doc.transact(+ () => doc.getText('source').insert(doc.getText('source').length, localEdit),+ 'agent',+ );+ expect(await storeManagedArtifactDoc(doc, docName, 'agent', ctx)).toBe('reconciled');+ }++ function backupDirs(): string[] {+ return readdirSync(resolve(home, '.ok', 'edit-backups', 'discarded')).sort();+ }++ function backupsIn(dir: string): string[] {+ const abs = resolve(home, '.ok', 'edit-backups', 'discarded', dir);+ return readdirSync(abs).map((f) => readFileSync(resolve(abs, f), 'utf-8'));+ }++ test('documents whose names differ only in non-Latin letters keep separate backups', async () => {+ await reconcileLocalEdit('__skill__/global/demo/references/会議', 'meeting edit');+ await reconcileLocalEdit('__skill__/global/demo/references/日報', 'daily edit');+ const dirs = backupDirs();+ expect(dirs).toHaveLength(2);+ const contents = dirs.map(backupsIn);+ expect(contents.map((c) => c.length)).toEqual([1, 1]);+ expect(contents.flat().some((c) => c.includes('meeting edit'))).toBe(true);+ expect(contents.flat().some((c) => c.includes('daily edit'))).toBe(true);+ });++ test('documents whose names differ only in punctuation keep separate backups', async () => {+ await reconcileLocalEdit('__skill__/global/demo/references/a b', 'space edit');+ await reconcileLocalEdit('__skill__/global/demo/references/a#b', 'hash edit');+ expect(backupDirs()).toHaveLength(2);+ });++ test('ASCII-safe documents keep the backup folder they had before', async () => {+ await reconcileLocalEdit('__skill__/global/demo/references/my_notes-v2.0', 'edit');+ expect(backupDirs()).toEqual(['__skill____global__demo__references__my_notes-v2.0']);+ });++ test('ASCII names that the separator encoding cannot tell apart keep separate backups', async () => {+ await reconcileLocalEdit('__skill__/global/demo/references/a/b', 'nested edit');+ await reconcileLocalEdit('__skill__/global/demo/references/a__b', 'underscore edit');+ await reconcileLocalEdit('__skill__/global/demo/references/a_/b', 'trailing edit');+ await reconcileLocalEdit('__skill__/global/demo/references/a/_b', 'leading edit');+ expect(backupDirs()).toHaveLength(4);+ });++ test('backup folder names stay filesystem-safe', async () => {+ await reconcileLocalEdit('__skill__/global/demo/references/会議 メモ', 'edit');+ for (const dir of backupDirs()) expect(dir).toMatch(/^[A-Za-z0-9._-]{1,120}$/);+ });+});+describe('concurrent-writer reconcile', () => {const docName = '__skill__/global/demo';diff --git a/packages/server/src/managed-artifact-persistence.ts b/packages/server/src/managed-artifact-persistence.tsindex cf98821e4..8d0a1db43 100644--- a/packages/server/src/managed-artifact-persistence.ts+++ b/packages/server/src/managed-artifact-persistence.ts@@ -4,12 +4,14 @@* re-canonicalize the markdown that gets projected verbatim into an agent's context).*/+import { createHash } from 'node:crypto';import { existsSync, readdirSync, readFileSync } from 'node:fs';import { homedir } from 'node:os';import { dirname, relative, resolve, sep } from 'node:path';import {LEGACY_SKILL_STORE_ROOT,LINEAGE_EPOCH_KEY,+ MANAGED_ARTIFACT_PREFIX_EXTSKILL,MANAGED_ARTIFACT_PREFIX_SKILL,MANAGED_ARTIFACT_PREFIX_TEMPLATE,type ManagedArtifactScope,@@ -253,6 +255,32 @@ export function loadManagedArtifactDoc(}const DISCARDED_EDIT_STASH_MAX = 5;+const DISCARDED_EDIT_READABLE_MAX = 100;+const DISCARDED_EDIT_DIGEST_LEN = 12;+const UNAMBIGUOUS_STASH_SEGMENT_RE = /^(?!.*__)[A-Za-z0-9.-](?:[A-Za-z0-9._-]*[A-Za-z0-9.-])?$/;++function discardedEditStashKey(documentName: string): string {+ const legacy = documentName.replace(/[^A-Za-z0-9._-]+/g, '__');+ const prefix = [+ MANAGED_ARTIFACT_PREFIX_SKILL,+ MANAGED_ARTIFACT_PREFIX_TEMPLATE,+ MANAGED_ARTIFACT_PREFIX_EXTSKILL,+ ].find((p) => documentName.startsWith(p));+ const rest = prefix === undefined ? documentName : documentName.slice(prefix.length);+ if (rest.split('/').every((segment) => UNAMBIGUOUS_STASH_SEGMENT_RE.test(segment))) {+ return legacy;+ }+ const canonical = documentName.normalize('NFC');+ const digest = createHash('sha256')+ .update(canonical)+ .digest('hex')+ .slice(0, DISCARDED_EDIT_DIGEST_LEN);+ const readable = canonical+ .replace(/[^A-Za-z0-9._-]+/g, '__')+ .replace(/^\.+/, '_')+ .slice(0, DISCARDED_EDIT_READABLE_MAX);+ return `${readable}-${digest}`;+}async function stashDiscardedEdit(documentName: string,@@ -260,8 +288,12 @@ async function stashDiscardedEdit(ctx: Pick<ManagedArtifactCtx, 'homedirOverride'>,): Promise<void> {try {- const safe = documentName.replace(/[^A-Za-z0-9._-]+/g, '__');- const dir = resolve(okUserHomeDir(homeFor(ctx)), 'edit-backups', 'discarded', safe);+ const dir = resolve(+ okUserHomeDir(homeFor(ctx)),+ 'edit-backups',+ 'discarded',+ discardedEditStashKey(documentName),+ );await tracedMkdir(dir, { recursive: true });const stamp = new Date().toISOString().replace(/[:.]/g, '-');await atomicWriteFile(resolve(dir, `${stamp}.md`), content, { fs: tracedAtomicFs });diff --git a/packages/server/src/shadow-repo.test.ts b/packages/server/src/shadow-repo.test.tsindex f5bdfe466..75229737c 100644--- a/packages/server/src/shadow-repo.test.ts+++ b/packages/server/src/shadow-repo.test.ts@@ -129,6 +129,49 @@ describe('initShadowRepo', () => {expect(existsSync(resolve(shadow2.gitDir, 'HEAD'))).toBe(true);});+ test('sub-projects whose names differ only in non-Latin letters get separate shadow repos', async () => {+ const repo = resolve(tmpDir, 'repo');+ mkdirSync(repo, { recursive: true });+ await simpleGit(repo).init();+ configureTestGitRepository(repo);+ const projectA = resolve(repo, 'ノート');+ const projectB = resolve(repo, '日記帳');+ mkdirSync(projectA, { recursive: true });+ mkdirSync(projectB, { recursive: true });++ const shadowA = await initShadowRepo(projectA);+ const shadowB = await initShadowRepo(projectB);++ expect(shadowA.gitDir).not.toBe(shadowB.gitDir);+ for (const [shadow, project] of [+ [shadowA, projectA],+ [shadowB, projectB],+ ] as const) {+ const sg = simpleGit().env({ GIT_DIR: shadow.gitDir });+ expect((await sg.raw('config', 'core.worktree')).trim()).toBe(project);+ }+ expect((await initShadowRepo(projectA)).gitDir).toBe(shadowA.gitDir);+ });++ test("an earlier version's lossy shadow repo stays with its owner after upgrade", async () => {+ const repo = resolve(tmpDir, 'repo');+ mkdirSync(repo, { recursive: true });+ await simpleGit(repo).init();+ configureTestGitRepository(repo);+ const owner = resolve(repo, 'ノート');+ const other = resolve(repo, '日記帳');+ mkdirSync(owner, { recursive: true });+ mkdirSync(other, { recursive: true });+ const legacyDir = resolve(repo, '.git', 'ok-___');+ execFileSync('git', ['init', '--bare', '--quiet', legacyDir]);+ execFileSync('git', ['config', 'core.worktree', owner], {+ env: { ...process.env, GIT_DIR: legacyDir },+ });++ expect((await initShadowRepo(owner)).gitDir).toBe(legacyDir);+ expect((await initShadowRepo(other)).gitDir).not.toBe(legacyDir);+ });+test('R9 rename shim: legacy .git/openknowledge/ is renamed to .git/ok/', async () => {const projectRoot = resolve(tmpDir, 'legacy');mkdirSync(projectRoot, { recursive: true });