Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Allow tags in any script, not just ASCII (PRD-9346) (#6235)

Andrew Mikofalvy04b6fef3d8a0
diff --git a/.changeset/unicode-tags.md b/.changeset/unicode-tags.md
new file mode 100644
index 000000000..420a8c50f
--- /dev/null
+++ b/.changeset/unicode-tags.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Tags can now use any script, including Japanese, Chinese, Korean, Cyrillic and accented Latin, in front matter and inline `#tags`. Existing non-ASCII tags that were hidden now appear in the tag list. Letters written directly after an inline tag name, such as `#wip日本語`, now extend that tag.
diff --git a/packages/app/src/components/ui/tag-pill-input.dom.test.tsx b/packages/app/src/components/ui/tag-pill-input.dom.test.tsx
index 119fe0cb5..411112a6c 100644
--- a/packages/app/src/components/ui/tag-pill-input.dom.test.tsx
+++ b/packages/app/src/components/ui/tag-pill-input.dom.test.tsx
@@ -78,6 +78,22 @@ describe('TagPillInput — input-side grammar gate', () => {
expect(alert?.textContent).toContain('Tags must start with a letter');
});
+ test('Enter, comma and Tab during IME composition do not commit', () => {
+ const onChange = vi.fn(() => {});
+ const { container } = renderInput({ onChange });
+ const input = container.querySelector('input') as HTMLInputElement;
+ fireEvent.change(input, { target: { value: 'にほんご' } });
+ fireEvent.keyDown(input, { key: 'Enter', isComposing: true });
+ fireEvent.keyDown(input, { key: ',', isComposing: true });
+ fireEvent.keyDown(input, { key: 'Tab', isComposing: true });
+ fireEvent.keyDown(input, { key: 'Enter', keyCode: 229 });
+ expect(onChange).toHaveBeenCalledTimes(0);
+ expect(input.value).toBe('にほんご');
+ fireEvent.change(input, { target: { value: '日本語' } });
+ fireEvent.keyDown(input, { key: 'Enter' });
+ expect(onChange).toHaveBeenCalledWith(['日本語']);
+ });
+
test('Enter on valid input commits + clears draft + clears any prior rejection state', () => {
const onChange = vi.fn(() => {});
const { container } = renderInput({ onChange });
diff --git a/packages/app/src/components/ui/tag-pill-input.tsx b/packages/app/src/components/ui/tag-pill-input.tsx
index 42ad125ef..fda35e508 100644
--- a/packages/app/src/components/ui/tag-pill-input.tsx
+++ b/packages/app/src/components/ui/tag-pill-input.tsx
@@ -9,6 +9,7 @@ import { XIcon } from 'lucide-react';
import { type Ref, useEffect, useId, useRef, useState } from 'react';
import { Badge } from '@/components/ui/badge';
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
+import { isImeCompositionKey } from '@/lib/ime-composition';
import { cn } from '@/lib/utils';
interface TagPillInputProps {
@@ -190,6 +191,7 @@ function TagPillInput({
if (highlightedEntry !== null) setHighlightedEntry(null);
}}
onKeyDown={(e) => {
+ if (isImeCompositionKey(e)) return;
const el = e.currentTarget;
const caretAtStart = el.selectionStart === 0 && el.selectionEnd === 0;
if (editingEntry === null && value.length > 0) {
diff --git a/packages/app/src/editor/components/TagView.dom.test.tsx b/packages/app/src/editor/components/TagView.dom.test.tsx
new file mode 100644
index 000000000..4669003dc
--- /dev/null
+++ b/packages/app/src/editor/components/TagView.dom.test.tsx
@@ -0,0 +1,55 @@
+import { cleanup, fireEvent, render } from '@testing-library/react';
+import type { NodeViewProps } from '@tiptap/core';
+import { afterEach, describe, expect, test } from 'vitest';
+import { TagView } from './TagView.tsx';
+
+afterEach(() => {
+ cleanup();
+});
+
+function renderPlaceholder(): HTMLInputElement {
+ const props = {
+ node: { attrs: { value: '' }, type: { name: 'tag' } },
+ getPos: () => undefined,
+ editor: {},
+ } as unknown as NodeViewProps;
+ const { container } = render(<TagView {...props} />);
+ return container.querySelector('input') as HTMLInputElement;
+}
+
+describe('TagView placeholder — IME composition', () => {
+ test('keeps an in-progress composition the tag grammar would reject', () => {
+ const input = renderPlaceholder();
+ fireEvent.compositionStart(input);
+ fireEvent.input(input, { target: { value: "ni'hao" }, isComposing: true });
+ expect(input.value).toBe("ni'hao");
+ });
+
+ test('a confirmed composition that is a valid tag stays in the draft', () => {
+ const input = renderPlaceholder();
+ fireEvent.compositionStart(input);
+ fireEvent.input(input, { target: { value: 'にほんご' }, isComposing: true });
+ fireEvent.input(input, { target: { value: '日本語' }, isComposing: true });
+ fireEvent.compositionEnd(input, { data: '日本語' });
+ expect(input.value).toBe('日本語');
+ });
+
+ test('a confirmed composition outside the grammar reverts to the last valid draft', () => {
+ const input = renderPlaceholder();
+ fireEvent.input(input, { target: { value: 'tag' } });
+ fireEvent.compositionStart(input);
+ fireEvent.input(input, { target: { value: 'tag。' }, isComposing: true });
+ fireEvent.compositionEnd(input, { target: { value: 'tag。' } });
+ expect(input.value).toBe('tag');
+ });
+
+ test('Enter and Space that confirm an IME candidate do not commit the tag', () => {
+ const input = renderPlaceholder();
+ fireEvent.input(input, { target: { value: 'にほん' } });
+ expect(fireEvent.keyDown(input, { key: 'Enter', isComposing: true })).toBe(true);
+ expect(fireEvent.keyDown(input, { key: ' ', isComposing: true })).toBe(true);
+ expect(fireEvent.keyDown(input, { key: 'Enter', keyCode: 229 })).toBe(true);
+ expect(input.value).toBe('にほん');
+ expect(fireEvent.keyDown(input, { key: 'Enter' })).toBe(false);
+ });
+});
diff --git a/packages/app/src/editor/components/TagView.tsx b/packages/app/src/editor/components/TagView.tsx
index 0bb37da08..49cf1f56a 100644
--- a/packages/app/src/editor/components/TagView.tsx
+++ b/packages/app/src/editor/components/TagView.tsx
@@ -4,6 +4,7 @@ import type { NodeViewProps } from '@tiptap/core';
import { TextSelection } from '@tiptap/pm/state';
import { NodeViewWrapper } from '@tiptap/react';
import { useEffect, useRef, useState } from 'react';
+import { isImeCompositionKey } from '@/lib/ime-composition';
function commitDraft(
editor: NodeViewProps['editor'],
@@ -56,8 +57,16 @@ function PlaceholderInput({ initialDraft, onCommit, onCancel }: PlaceholderInput
const { t } = useLingui();
const inputRef = useRef<HTMLInputElement>(null);
const [draft, setDraft] = useState(initialDraft);
+ const lastValidDraftRef = useRef(initialDraft);
const committedRef = useRef(false);
+ const acceptDraft = (next: string): boolean => {
+ if (next !== '' && !INLINE_TAG_VALUE_RE.test(next)) return false;
+ lastValidDraftRef.current = next;
+ setDraft(next);
+ return true;
+ };
+
useEffect(() => {
const handle = requestAnimationFrame(() => {
inputRef.current?.focus();
@@ -86,11 +95,17 @@ function PlaceholderInput({ initialDraft, onCommit, onCancel }: PlaceholderInput
className="bg-transparent border-0 p-0 outline-none focus:outline-none focus:ring-0 text-inherit font-inherit"
onChange={(e) => {
const next = e.target.value;
- if (next === '' || INLINE_TAG_VALUE_RE.test(next)) {
+ if ((e.nativeEvent as InputEvent).isComposing) {
setDraft(next);
+ return;
}
+ acceptDraft(next);
+ }}
+ onCompositionEnd={(e) => {
+ if (!acceptDraft(e.currentTarget.value)) setDraft(lastValidDraftRef.current);
}}
onKeyDown={(e) => {
+ if (isImeCompositionKey(e)) return;
if (e.key === 'Enter' || e.key === ' ') {
e.preventDefault();
committedRef.current = true;
diff --git a/packages/app/src/editor/extensions/tag-suggestion.test.ts b/packages/app/src/editor/extensions/tag-suggestion.test.ts
index 4b6f0ddc4..444d8dabd 100644
--- a/packages/app/src/editor/extensions/tag-suggestion.test.ts
+++ b/packages/app/src/editor/extensions/tag-suggestion.test.ts
@@ -82,6 +82,12 @@ describe('buildTagSuggestionItems — filter + rank', () => {
expect(out).toEqual([]);
});
+ test('non-ASCII tag names offer a Create row', () => {
+ const out = buildTagSuggestionItems(tags([['日本', 2]]), '日本語');
+ expect(out[out.length - 1]).toEqual({ kind: 'create', value: '日本語' });
+ expect(buildTagSuggestionItems(tags([]), 'café')).toEqual([{ kind: 'create', value: 'café' }]);
+ });
+
test('hierarchical tag names work end-to-end', () => {
const out = buildTagSuggestionItems(
tags([
@@ -204,6 +210,15 @@ describe('tagMatcher — boundary semantics', () => {
expect(tagMatcher(stubPosition('#-bad', 1) as never)).toBeNull();
});
+ test('`#日本語` triggers with the full non-ASCII query', () => {
+ const result = tagMatcher(stubPosition('メモ #日本語', 1) as never);
+ expect(result).toEqual({
+ range: { from: 4, to: 8 },
+ query: '日本語',
+ text: '#日本語',
+ });
+ });
+
test('matches `#tag` after an inline-atom placeholder (textBetween emits )', () => {
const result = tagMatcher(stubPosition('#wip', 1) as never);
expect(result).toEqual({
diff --git a/packages/app/src/editor/extensions/tag-suggestion.ts b/packages/app/src/editor/extensions/tag-suggestion.ts
index f14057e64..148db3413 100644
--- a/packages/app/src/editor/extensions/tag-suggestion.ts
+++ b/packages/app/src/editor/extensions/tag-suggestion.ts
@@ -1,3 +1,7 @@
+import {
+ INLINE_TAG_VALUE_PATTERN_SOURCE,
+ INLINE_TAG_VALUE_RE,
+} from '@inkeep/open-knowledge-core/markdown/tag-promotion';
import type { Editor } from '@tiptap/core';
import type { ResolvedPos } from '@tiptap/pm/model';
import { PluginKey } from '@tiptap/pm/state';
@@ -25,7 +29,7 @@ export type TagSuggestionItem =
const MAX_ITEMS = 8;
-const TAG_VALID_RE = /^[a-zA-Z][\w/-]*$/;
+const TAG_TRIGGER_RE = new RegExp(`(^|[\\s])#(${INLINE_TAG_VALUE_PATTERN_SOURCE})?$`, 'u');
export async function fetchTags(): Promise<TagSummaryEntry[]> {
const r = await fetch('/api/tags');
@@ -65,7 +69,7 @@ export function buildTagSuggestionItems(
}));
const trimmed = query.trim();
- if (trimmed && TAG_VALID_RE.test(trimmed) && !tags.some((t) => t.name === trimmed)) {
+ if (trimmed && INLINE_TAG_VALUE_RE.test(trimmed) && !tags.some((t) => t.name === trimmed)) {
items.push({ kind: 'create', value: trimmed });
}
@@ -78,7 +82,7 @@ export function tagMatcher(config: {
const { $position } = config;
const textBefore = $position.parent.textBetween(0, $position.parentOffset, undefined, '');
- const match = textBefore.match(/(^|[\s])#([a-zA-Z][\w/-]*)?$/);
+ const match = textBefore.match(TAG_TRIGGER_RE);
if (!match) return null;
const query = match[2] ?? '';
@@ -130,7 +134,7 @@ export function configureTagSuggestion(editor: Editor) {
command: ({ editor, range, props: item }) => {
try {
const value = item.value;
- if (!value || !TAG_VALID_RE.test(value)) return;
+ if (!value || !INLINE_TAG_VALUE_RE.test(value)) return;
editor
.chain()
.focus()
@@ -241,7 +245,7 @@ export function configureTagSuggestion(editor: Editor) {
return true;
}
const trimmed = (currentProps.query ?? '').trim();
- if (trimmed && TAG_VALID_RE.test(trimmed)) {
+ if (trimmed && INLINE_TAG_VALUE_RE.test(trimmed)) {
currentProps.command({ kind: 'create', value: trimmed });
return true;
}
diff --git a/packages/app/src/lib/ime-composition.dom.test.tsx b/packages/app/src/lib/ime-composition.dom.test.tsx
new file mode 100644
index 000000000..e2de56b50
--- /dev/null
+++ b/packages/app/src/lib/ime-composition.dom.test.tsx
@@ -0,0 +1,28 @@
+import { cleanup, fireEvent, render } from '@testing-library/react';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { isImeCompositionKey } from './ime-composition.ts';
+
+afterEach(() => {
+ cleanup();
+});
+
+function keyResult(init: KeyboardEventInit & { keyCode?: number }): boolean {
+ const seen = vi.fn((composing: boolean) => composing);
+ const { container } = render(<input onKeyDown={(e) => seen(isImeCompositionKey(e))} />);
+ fireEvent.keyDown(container.querySelector('input') as HTMLInputElement, init);
+ return seen.mock.results[0]?.value as boolean;
+}
+
+describe('isImeCompositionKey', () => {
+ test('is true while a composition is active', () => {
+ expect(keyResult({ key: 'Enter', isComposing: true })).toBe(true);
+ });
+
+ test("is true for Safari's confirming Enter (keyCode 229, isComposing false)", () => {
+ expect(keyResult({ key: 'Enter', keyCode: 229 })).toBe(true);
+ });
+
+ test('is false for an ordinary Enter', () => {
+ expect(keyResult({ key: 'Enter', keyCode: 13 })).toBe(false);
+ });
+});
diff --git a/packages/app/src/lib/ime-composition.ts b/packages/app/src/lib/ime-composition.ts
new file mode 100644
index 000000000..373b66bfb
--- /dev/null
+++ b/packages/app/src/lib/ime-composition.ts
@@ -0,0 +1,11 @@
+import type { KeyboardEvent } from 'react';
+
+const SAFARI_IME_CONFIRM_KEY_CODE = 229;
+
+export function isImeCompositionKey(event: KeyboardEvent): boolean {
+ return (
+ event.nativeEvent.isComposing ||
+ // oxlint-disable-next-line typescript/no-deprecated -- Safari fires the IME-confirming Enter after compositionend with isComposing false; keyCode 229 is the only signal it carries
+ event.keyCode === SAFARI_IME_CONFIRM_KEY_CODE
+ );
+}
diff --git a/packages/core/src/frontmatter/tags.test.ts b/packages/core/src/frontmatter/tags.test.ts
index 109a4736f..9ad25fc40 100644
--- a/packages/core/src/frontmatter/tags.test.ts
+++ b/packages/core/src/frontmatter/tags.test.ts
@@ -92,6 +92,18 @@ describe('extractFrontmatterTags', () => {
const yaml = 'tags:\n - valid\n - [inner, list]\n - alsoValid\n';
expect(extractFrontmatterTags(yaml)).toEqual(['valid', 'alsoValid']);
});
+ test('keeps non-ASCII tags instead of dropping them', () => {
+ expect(extractFrontmatterTags('tags: [蔵, 心筋梗塞, 循環器]\n')).toEqual([
+ '蔵',
+ '心筋梗塞',
+ '循環器',
+ ]);
+ expect(extractFrontmatterTags('tags: [café, "#Привет", 研究/循環器]\n')).toEqual([
+ 'café',
+ 'Привет',
+ '研究/循環器',
+ ]);
+ });
});
describe('isValidFrontmatterTagValue', () => {
@@ -148,6 +160,40 @@ describe('isValidFrontmatterTagValue', () => {
expect(isValidFrontmatterTagValue('##showcase')).toBe(false);
});
+ test('accepts letters and digits from every script, including combining marks', () => {
+ for (const value of [
+ '蔵',
+ '心筋梗塞',
+ 'カテゴリー',
+ '한국어',
+ 'Привет',
+ 'Ελληνικά',
+ 'العربية',
+ 'हिन्दी',
+ 'ภาษาไทย',
+ 'café',
+ 'cafe\u0301',
+ 'naïve/ü-ber_2026',
+ '2026',
+ ]) {
+ expect(isValidFrontmatterTagValue(value)).toBe(true);
+ }
+ });
+
+ test('rejects a leading combining mark and invisible or bidi control characters', () => {
+ expect(isValidFrontmatterTagValue('\u0301cafe')).toBe(false);
+ expect(isValidFrontmatterTagValue('ta\u200bg')).toBe(false);
+ expect(isValidFrontmatterTagValue('ta\u200dg')).toBe(false);
+ expect(isValidFrontmatterTagValue('\u202etag')).toBe(false);
+ expect(isValidFrontmatterTagValue('\u3164')).toBe(false);
+ expect(isValidFrontmatterTagValue('tag\ufe0f')).toBe(false);
+ expect(isValidFrontmatterTagValue('ta\u034fg')).toBe(false);
+ expect(isValidFrontmatterTagValue('葛\u{e0100}')).toBe(false);
+ expect(isValidFrontmatterTagValue('tag\u00a0two')).toBe(false);
+ expect(isValidFrontmatterTagValue('日本 語')).toBe(false);
+ expect(isValidFrontmatterTagValue('タグ。')).toBe(false);
+ });
+
test('agrees with the FRONTMATTER_TAG_VALUE_RE constant (single source of truth)', () => {
for (const value of ['showcase', '2026', 'hello world', '']) {
const stripped = value.startsWith('#') ? value.slice(1) : value;
diff --git a/packages/core/src/frontmatter/tags.ts b/packages/core/src/frontmatter/tags.ts
index 764b26739..34adb090a 100644
--- a/packages/core/src/frontmatter/tags.ts
+++ b/packages/core/src/frontmatter/tags.ts
@@ -1,6 +1,10 @@
+import { TAG_CHAR_PATTERN_SOURCE } from '../markdown/tag-promotion.ts';
import { parseFrontmatterYaml } from './yaml-codec.ts';
-export const FRONTMATTER_TAG_VALUE_RE = /^[a-zA-Z0-9][\w/-]*$/;
+export const FRONTMATTER_TAG_VALUE_RE = new RegExp(
+ `^(?!\\p{DI})[\\p{L}\\p{N}]${TAG_CHAR_PATTERN_SOURCE}*$`,
+ 'u',
+);
export function isValidFrontmatterTagValue(value: string): boolean {
if (typeof value !== 'string') return false;
diff --git a/packages/core/src/markdown/tag-promotion.ts b/packages/core/src/markdown/tag-promotion.ts
index 59a327a18..eff16c6f7 100644
--- a/packages/core/src/markdown/tag-promotion.ts
+++ b/packages/core/src/markdown/tag-promotion.ts
@@ -7,13 +7,16 @@ import {
sliceTextWithProvenance,
} from './promoter-position.ts';
-export const TAG_IN_TEXT_PATTERN_SOURCE = '(^|\\s)#([a-zA-Z][\\w/-]*)';
+const TAG_CHAR_CLASS_SOURCE = '[\\p{L}\\p{M}\\p{N}_/-]';
+export const TAG_CHAR_PATTERN_SOURCE = `(?:(?!\\p{DI})${TAG_CHAR_CLASS_SOURCE})`;
+export const INLINE_TAG_VALUE_PATTERN_SOURCE = `(?!\\p{DI})\\p{L}${TAG_CHAR_PATTERN_SOURCE}*`;
+export const TAG_IN_TEXT_PATTERN_SOURCE = `(^|\\s)#(${INLINE_TAG_VALUE_PATTERN_SOURCE})(?!\\p{DI}*${TAG_CHAR_CLASS_SOURCE})`;
export function createTagInTextRegex(): RegExp {
- return new RegExp(TAG_IN_TEXT_PATTERN_SOURCE, 'g');
+ return new RegExp(TAG_IN_TEXT_PATTERN_SOURCE, 'gu');
}
const TAG_IN_TEXT_RE = createTagInTextRegex();
-export const INLINE_TAG_VALUE_RE = /^[a-zA-Z][\w/-]*$/;
+export const INLINE_TAG_VALUE_RE = new RegExp(`^${INLINE_TAG_VALUE_PATTERN_SOURCE}$`, 'u');
export function promoteTagsInParent(parent: Parent, source: string = ''): void {
const newChildren: Parent['children'] = [];
diff --git a/packages/core/src/markdown/tag-to-markdown.ts b/packages/core/src/markdown/tag-to-markdown.ts
index f3847a26b..480234f3e 100644
--- a/packages/core/src/markdown/tag-to-markdown.ts
+++ b/packages/core/src/markdown/tag-to-markdown.ts
@@ -17,7 +17,9 @@ export const tagToMarkdown: {
}>;
} = {
handlers: { tag: tagHandler },
- unsafe: [{ character: '#', inConstruct: ['phrasing'], before: '\\s', after: '[a-zA-Z]' }],
+ unsafe: [
+ { character: '#', inConstruct: ['phrasing'], before: '\\s', after: '[A-Za-z]|[^\\x00-\\x7F]' },
+ ],
};
export function remarkTags(this: Processor) {
diff --git a/packages/server/src/tag-index.persistence.test.ts b/packages/server/src/tag-index.persistence.test.ts
index c4e283ef7..f6ccf755e 100644
--- a/packages/server/src/tag-index.persistence.test.ts
+++ b/packages/server/src/tag-index.persistence.test.ts
@@ -47,7 +47,7 @@ describe('TagIndex persistence', () => {
await idx.saveToDisk();
expect(existsSync(rig.snapshotPath)).toBe(true);
const snapshot = JSON.parse(await readFile(rig.snapshotPath, 'utf-8'));
- expect(snapshot.version).toBe(1);
+ expect(snapshot.version).toBe(2);
expect(snapshot.docs.alpha).toEqual(['typescript']);
expect(snapshot.files.alpha.mtimeMs).toBeGreaterThan(0);
expect(snapshot.files.alpha.size).toBeGreaterThan(0);
@@ -78,6 +78,30 @@ describe('TagIndex persistence', () => {
}
});
+ test('a snapshot written under the ASCII-only grammar is rebuilt so unchanged non-ASCII tags appear', async () => {
+ const rig = tempRig();
+ try {
+ const body = '---\ntags: [蔵, 心筋梗塞]\n---\nBody.\n';
+ writeDoc(rig.contentDir, 'alpha.md', body, 1_000);
+ mkdirSync(dirname(rig.snapshotPath), { recursive: true });
+ writeFileSync(
+ rig.snapshotPath,
+ JSON.stringify({
+ version: 1,
+ docs: { alpha: [] },
+ files: { alpha: { mtimeMs: 1_000_000, size: Buffer.byteLength(body) } },
+ }),
+ );
+ const idx = newIndex(rig);
+ expect(await idx.loadFromDisk()).toBe(false);
+ await idx.init();
+ expect(idx.getDocsForTag('蔵')).toEqual(['alpha']);
+ expect(idx.getDocsForTag('心筋梗塞')).toEqual(['alpha']);
+ } finally {
+ rig.cleanup();
+ }
+ });
+
test('warm boot with offline edits, adds, and deletes reconciles all three', async () => {
const rig = tempRig();
try {
@@ -205,10 +229,11 @@ describe('TagIndex persistence', () => {
test.each([
{ snapshot: '[]', kept: false },
- { snapshot: '{"version":1,"docs":{"a":"not-an-array"},"files":{}}', kept: false },
- { snapshot: '{"version":1,"docs":{"a":[""]},"files":{}}', kept: false },
- { snapshot: '{"version":1,"docs":{},"files":{"a":{"mtimeMs":"nan","size":1}}}', kept: false },
- { snapshot: '{"version":2,"docs":{},"files":{}}', kept: true },
+ { snapshot: '{"version":2,"docs":{"a":"not-an-array"},"files":{}}', kept: false },
+ { snapshot: '{"version":2,"docs":{"a":[""]},"files":{}}', kept: false },
+ { snapshot: '{"version":2,"docs":{},"files":{"a":{"mtimeMs":"nan","size":1}}}', kept: false },
+ { snapshot: '{"version":1,"docs":{},"files":{}}', kept: true },
+ { snapshot: '{"version":3,"docs":{},"files":{}}', kept: true },
])('off-shape snapshot $snapshot reports a miss (kept: $kept)', async ({ snapshot, kept }) => {
const rig = tempRig();
try {
@@ -229,7 +254,7 @@ describe('TagIndex persistence', () => {
writeFileSync(
rig.snapshotPath,
JSON.stringify({
- version: 1,
+ version: 2,
docs: { __system__: ['smuggled'], alpha: ['legit'] },
files: {},
}),
diff --git a/packages/server/src/tag-index.test.ts b/packages/server/src/tag-index.test.ts
index 5329ae08e..b504a78cf 100644
--- a/packages/server/src/tag-index.test.ts
+++ b/packages/server/src/tag-index.test.ts
@@ -41,6 +41,26 @@ describe('TagIndex', () => {
}
});
+ test('indexes non-ASCII frontmatter and inline tags', () => {
+ const { contentDir, cleanup } = tempContentDir();
+ try {
+ const idx = new TagIndex({ contentDir });
+ idx.updateDocumentFromMarkdown(
+ 'alpha',
+ '---\ntags: [蔵, 心筋梗塞, 循環器]\n---\nBody #循環器/不整脈 and #café.\n',
+ );
+ for (const tag of ['蔵', '心筋梗塞', '循環器', '循環器/不整脈', 'café']) {
+ expect(idx.getDocsForTag(tag)).toEqual(['alpha']);
+ }
+ expect(idx.getDocsForTag('caf')).toEqual([]);
+ idx.updateDocumentFromMarkdown('beta', 'Ward #葛\u{e0100}飾区 and #ta\u034fg.\n');
+ expect(idx.getAllTags().map((t) => t.name)).not.toContain('葛');
+ expect(idx.getDocsForTag('ta')).toEqual([]);
+ } finally {
+ cleanup();
+ }
+ });
+
test('merges inline + frontmatter tags into the same index', () => {
const { contentDir, cleanup } = tempContentDir();
try {
diff --git a/packages/server/src/tag-index.ts b/packages/server/src/tag-index.ts
index ba3a55037..eb1a55fcf 100644
--- a/packages/server/src/tag-index.ts
+++ b/packages/server/src/tag-index.ts
@@ -41,12 +41,12 @@ interface FileMeta {
}
interface SerializedTagIndexSnapshot {
- version: 1;
+ version: 2;
docs: Record<string, string[]>;
files: Record<string, FileMeta>;
}
-const SNAPSHOT_VERSION = 1;
+const SNAPSHOT_VERSION = 2;
function isOtherSnapshotVersion(data: unknown): boolean {
if (typeof data !== 'object' || data === null) return false;