Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): index non-Latin scripts in workspace search (#3322)
mike-inkeep2a6cbe5ac517
diff --git a/.changeset/non-latin-search-tokenizer.md b/.changeset/non-latin-search-tokenizer.mdnew file mode 100644index 000000000..86a603096--- /dev/null+++ b/.changeset/non-latin-search-tokenizer.md@@ -0,0 +1,5 @@+---+"@inkeep/open-knowledge": patch+---++Workspace search now finds content written in non-Latin scripts. Previously the search index kept only Latin-script words, so document bodies in Hebrew, Cyrillic, Greek, Arabic, Thai, Chinese, Japanese, Korean, and other scripts indexed to zero searchable words — the omnibar and the MCP `search` tool couldn't find them at all, and in a mixed-script query only the Latin words did any work. Search now additionally segments those scripts with the runtime's built-in word segmenter (dictionary-based for unspaced scripts like Thai, Japanese, and Chinese), with no new dependencies and no change to how Latin-only content is matched or ranked. In scripts where a single character carries a whole syllable — Chinese, Japanese, Korean, Thai, Khmer — queries are matched exactly rather than with the one-character typo tolerance applied elsewhere, since there a one-character difference is a different word and near-misses would crowd out the exact match. Thanks to @Guohao1020 for the report, root-cause analysis, and proposed approach, and to @AsafDafna for the implementation and validation (#903).diff --git a/packages/core/src/search/workspace-search.non-latin.test.ts b/packages/core/src/search/workspace-search.non-latin.test.tsnew file mode 100644index 000000000..317e31314--- /dev/null+++ b/packages/core/src/search/workspace-search.non-latin.test.ts@@ -0,0 +1,263 @@+import { describe, expect, test, vi } from 'vitest';+import {+ createWorkspaceSearchCorpus,+ createWorkspaceSearchDocument,+ searchWorkspaceCorpus,+} from './workspace-search.ts';++/**+ * Non-Latin body search (inkeep/open-knowledge#903).+ *+ * Orama's default `english` tokenizer treats every character outside its Latin+ * class as a separator, so non-Latin body content indexed to ZERO tokens and+ * was unsearchable — masked for file NAMES by the lexical ladder, which+ * compares whole strings. These tests search by BODY content behind+ * Latin-named docs, so a hit can only come from the BM25 index and a+ * tokenizer regression cannot hide behind the name ladder.+ */++const docs = [+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/coal.md',+ title: 'coal notes',+ content: 'הפקת פחם באירופה במאה התשע עשרה',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/access.md',+ title: 'access notes',+ content: '访问控制与权限管理的最佳实践',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/tokyo.md',+ title: 'tokyo notes',+ content: '検索テキストの品質を確認します',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/tenants.md',+ title: 'tenant notes',+ content: 'поиск работает: tenant isolation details',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/thai.md',+ title: 'thai notes',+ content: 'ค้นหาข้อความภาษาไทย',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/korean.md',+ title: 'korean notes',+ content: '검색 품질을 확인합니다',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/avalanche.md',+ title: 'avalanche notes',+ content: 'çığ düştü ve yol kapandı',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/plain.md',+ title: 'plain notes',+ content: 'a plain english document about foxes',+ }),+];++const corpus = createWorkspaceSearchCorpus(docs);++function pathsFor(query: string): string[] {+ return searchWorkspaceCorpus(corpus, query, { intent: 'full_text' }).map(+ (result) => result.document.path,+ );+}++describe('non-Latin body content is searchable', () => {+ test.each([+ ['Hebrew', 'פחם', 'notes/coal.md'],+ ['Chinese', '访问控制', 'notes/access.md'],+ ['Japanese', '検索テキスト', 'notes/tokyo.md'],+ ['Cyrillic', 'поиск работает', 'notes/tenants.md'],+ ['Thai', 'ค้นหา', 'notes/thai.md'],+ ['Korean', '검색', 'notes/korean.md'],+ ])('%s query finds body content', (_script, query, expectedPath) => {+ expect(pathsFor(query)).toContain(expectedPath);+ });++ test('accented Latin outside the default set is searchable', () => {+ // Latin-script letters outside the splitter's kept set (`àèéìòóù`) are+ // separators too. `çığ` is made only of those, so the default tokenizer+ // reduces it to ZERO tokens on both sides and it is unfindable — the same+ // failure as the non-Latin scripts above, in Latin script. Words that keep+ // some fragment (`niño` → `ni`/`o`) stay findable either way, so this+ // all-dropped case is what actually pins the behavior.+ expect(pathsFor('çığ')).toContain('notes/avalanche.md');+ });++ test('mixed-script query uses both halves', () => {+ // The Cyrillic half alone must be able to select the doc: `поиск` appears+ // only in tenants.md, so a tokenizer that silently drops the non-Latin+ // half would still pass a combined-terms assertion — pin the ranking too.+ const paths = pathsFor('поиск isolation');+ expect(paths[0]).toBe('notes/tenants.md');+ expect(pathsFor('поиск')).toContain('notes/tenants.md');+ });++ test('the omnibar call shape (full_text candidates, navigation ranking) finds non-Latin content', () => {+ const results = searchWorkspaceCorpus(corpus, '访问控制', {+ intent: 'full_text',+ ranking: 'navigation',+ });+ expect(results.map((result) => result.document.path)).toContain('notes/access.md');+ });++ test('pure-Latin queries are unaffected', () => {+ expect(pathsFor('foxes')[0]).toBe('notes/plain.md');+ });+});++describe('dense scripts do not take edit-distance tolerance', () => {+ // Now that these scripts tokenize at all, the `full_text` tolerance of 1+ // reaches them for the first time. A word there is a few characters long, so+ // one edit is a different word, not a typo — and BM25 can float that+ // near-miss above the exact hit.+ const cjk = createWorkspaceSearchCorpus([+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/access.md',+ title: 'access notes',+ content: '访问控制与权限管理',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/variant.md',+ title: 'variant notes',+ // `防问控制` — one character off, and not a word.+ content: '防问控制与权限管理',+ }),+ // A genuinely related doc. It legitimately ranks first (shorter, and it+ // really does contain 访问 + 控制), and its presence is what shifts the+ // corpus statistics far enough for the near-miss to beat the exact hit.+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/controller.md',+ title: 'controller notes',+ content: '访问控制器的配置说明',+ }),+ ]);++ test('a one-character variant does not outrank the exact match', () => {+ const paths = searchWorkspaceCorpus(cjk, '访问控制', { intent: 'full_text' }).map(+ (result) => result.document.path,+ );+ expect(paths.indexOf('notes/access.md')).toBeLessThan(paths.indexOf('notes/variant.md'));+ });++ // Hangul is space-separated, so it is NOT an unspaced script — but a syllable+ // block carries as much as a whole Latin word, so it fails the same way:+ // `검색` (search) vs `검토` (review) vs `검사` (inspection) are pairwise one+ // edit apart and mean different things.+ const hangul = createWorkspaceSearchCorpus([+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/search.md',+ title: 'search notes',+ content: '검색 품질을 확인합니다',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/review.md',+ title: 'review notes',+ content: '검토 품질을 확인합니다',+ }),+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/inspection.md',+ title: 'inspection notes',+ content: '검사 결과를 확인합니다',+ }),+ ]);++ test('a one-syllable Hangul variant does not outrank the exact match', () => {+ const paths = searchWorkspaceCorpus(hangul, '검색 품질', { intent: 'full_text' }).map(+ (result) => result.document.path,+ );+ expect(paths[0]).toBe('notes/search.md');+ });++ // The other half of the contract: alphabetic non-Latin scripts must KEEP+ // tolerance, so widening the dense-script ranges over them is caught. The+ // recall tests above all query exact terms and would survive that widening.+ const cyrillic = createWorkspaceSearchCorpus([+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/russian.md',+ title: 'russian notes',+ content: 'поиск работает хорошо',+ }),+ ]);++ test('a one-letter Cyrillic typo still finds the document', () => {+ // `поиек` (е for с) appears nowhere in the corpus — only tolerance finds it.+ expect(+ searchWorkspaceCorpus(cyrillic, 'поиек', { intent: 'full_text' }).map(+ (result) => result.document.path,+ ),+ ).toContain('notes/russian.md');+ });+});++describe('input the base tokenizer already handles stays untouched', () => {+ function topScoreFor(content: string): number {+ const single = createWorkspaceSearchCorpus([+ createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/school.md',+ title: 'school notes',+ content,+ }),+ ]);+ const results = searchWorkspaceCorpus(single, 'ecole', { intent: 'full_text' });+ expect(results).not.toHaveLength(0);+ return results[0].score;+ }++ test('capitalizing an accented word does not change its score', () => {+ // Capitalization is not a script difference: the base lowercases before it+ // splits, so `École` and `école` yield the same base tokens (`ecole`,+ // diacritic-folded). Treating `É` as dropped would append a second,+ // unfolded `école` to the capitalized doc only — same hits, but a longer+ // field and therefore a different BM25 score.+ expect(topScoreFor('École normale')).toBe(topScoreFor('école normale'));+ });++ test('a runtime without Intl.Segmenter degrades instead of failing module load', async () => {+ // This module is imported by the command palette and the wiki-link+ // suggestion extension, so an unguarded constructor would take down the+ // app bundle's module evaluation, not just non-Latin search.+ const descriptor = Object.getOwnPropertyDescriptor(Intl, 'Segmenter');+ Reflect.deleteProperty(Intl, 'Segmenter');+ vi.resetModules();+ try {+ const mod = await import('./workspace-search.ts');+ const fallbackCorpus = mod.createWorkspaceSearchCorpus([+ mod.createWorkspaceSearchDocument({+ kind: 'page',+ path: 'notes/plain.md',+ title: 'plain notes',+ content: 'a plain english document about foxes',+ }),+ ]);+ const paths = mod+ .searchWorkspaceCorpus(fallbackCorpus, 'foxes', { intent: 'full_text' })+ .map((result) => result.document.path);+ expect(paths).toContain('notes/plain.md');+ } finally {+ if (descriptor) Object.defineProperty(Intl, 'Segmenter', descriptor);+ vi.resetModules();+ }+ });+});diff --git a/packages/core/src/search/workspace-search.ts b/packages/core/src/search/workspace-search.tsindex f03433a16..a9072338a 100644--- a/packages/core/src/search/workspace-search.ts+++ b/packages/core/src/search/workspace-search.ts@@ -1,4 +1,5 @@import { type AnyOrama, count, create, insertMultiple, removeMultiple, search } from '@orama/orama';+import { tokenizer as oramaTokenizer } from '@orama/orama/components';import { isHiddenDocName } from '../util/doc-name.ts';export type WorkspaceSearchKind = 'page' | 'folder' | 'file';@@ -253,10 +254,96 @@ export function createWorkspaceSearchDocument(input: {};}+/**+ * Matches any letter or number that Orama's default `english` tokenizer treats+ * as a separator rather than a word character. The default splitter+ * (`/[^A-Za-zàèéìòóù0-9_'-]+/`) keeps only that Latin class, so a run of+ * Hebrew, Cyrillic, Greek, Arabic, CJK, Thai, … is deleted wholesale on both+ * the indexing and the query side — such content indexes to zero tokens and is+ * unsearchable. Read the double negation as: a Unicode letter NOT in the+ * default-kept Latin set, or a Unicode number NOT in `0-9`.+ *+ * Case-insensitive because the base splits `input.toLowerCase()`, so the kept+ * set is really a set of case-folded letters: `É` is kept (it folds to the kept+ * `é`) and must NOT count as dropped, or `École` would index the base's+ * diacritic-folded `ecole` AND an appended `école`, inflating `fieldLength`+ * for input the base handles fine. Folding also covers lookalikes that fold+ * into the set, such as the Kelvin sign `K` → `k`.+ *+ * The number half admits only `Nd` and `Nl` — the number categories whose+ * characters segment as word-like. `No` (`½`, `²`, `¾`) never does, so+ * admitting it would route a whole document through the segmenter to append+ * nothing: a single `m²` used to double that document's index-build cost+ * (2000 Latin docs / 1.5 MB: 93 ms admitting `No`, 46 ms not, same tokens).+ *+ * Cost when the gate does fire is the segmentation itself, on the same event+ * loop as the index build: for 2000 docs it is 91 ms Hebrew (3.4 MB), 140 ms+ * Thai (6.0 MB), and 163 ms Chinese (3.0 MB), against 78 ms for a Latin corpus+ * that never reaches the segmenter.+ */+const DROPPED_SCRIPT_CHAR = /[^\P{L}a-zàèéìòóù]|[^\P{Nd}0-9]|\p{Nl}/iu;++/**+ * Word segmenter for scripts the default tokenizer drops: plain word splitting+ * for spaced scripts (Hebrew, Cyrillic, Greek, Arabic), dictionary-based+ * segmentation for unspaced ones (Thai, Japanese, Chinese). Ships with every+ * supported runtime (Node ≥ 16, Electron, evergreen browsers) — no dependency.+ * The locale is pinned so token output never varies with the host machine's+ * default locale: word-granularity segmentation dispatches by the script of+ * the text itself, so the pinned value costs nothing for non-English content,+ * and index/query tokenization stays deterministic across environments (and in+ * token-level tests). Stateless, so one shared instance serves every index.+ *+ * Feature-detected rather than constructed outright: this module is imported by+ * the command palette and the wiki-link suggestion extension, so a throw here+ * would fail the app bundle's module evaluation on a runtime without the API,+ * not merely degrade non-Latin search. `null` falls back to base tokenization,+ * i.e. Latin-only search.+ */+const WORD_SEGMENTER = (() => {+ const Ctor = (globalThis as { Intl: { Segmenter?: typeof Intl.Segmenter } }).Intl.Segmenter;+ return Ctor ? new Ctor('en', { granularity: 'word' }) : null;+})();++/**+ * The default Orama tokenizer, extended to ALSO emit tokens for scripts the+ * default splitter deletes. Latin-only input (no {@link DROPPED_SCRIPT_CHAR})+ * returns the base result unchanged — same tokens, same order — so existing+ * tokenization and BM25 scores are byte-identical and the pinned flag-OFF+ * parity fixture keeps passing. Input containing a dropped script additionally+ * yields that script's `Intl.Segmenter` word segments, lowercased to match the+ * base pipeline and deduplicated like the base's `allowDuplicates: false`.+ * Orama routes both indexing and querying through this tokenizer, so the two+ * sides stay consistent.+ */+function createWorkspaceSearchTokenizer() {+ const base = oramaTokenizer.createTokenizer({ language: 'english' });+ const baseTokenize = base.tokenize.bind(base);+ base.tokenize = (input, language, prop, withCache) => {+ const tokens = baseTokenize(input, language, prop, withCache);+ if (!WORD_SEGMENTER || typeof input !== 'string') return tokens;+ if (!DROPPED_SCRIPT_CHAR.test(input)) return tokens;+ const seen = new Set(tokens);+ for (const { segment, isWordLike } of WORD_SEGMENTER.segment(input)) {+ if (!isWordLike || !DROPPED_SCRIPT_CHAR.test(segment)) continue;+ const token = segment.toLowerCase();+ if (!seen.has(token)) {+ seen.add(token);+ tokens.push(token);+ }+ }+ return tokens;+ };+ return base;+}+function createWorkspaceSearchIndex(documents: readonly WorkspaceSearchDocument[],): WorkspaceSearchIndex {- const db = create({ schema: WORKSPACE_SEARCH_SCHEMA });+ const db = create({+ schema: WORKSPACE_SEARCH_SCHEMA,+ components: { tokenizer: createWorkspaceSearchTokenizer() },+ });if (documents.length > 0) {insertMultiple(db, documents as WorkspaceSearchDocument[]);}@@ -663,9 +750,36 @@ function searchBoost(return { title: 8, name: 7, path: 5, pathSegments: 4 };}+/**+ * Scripts where one character carries a whole syllable or morpheme, so a word+ * is only a few characters long: CJK, Thai, Khmer, and Hangul. An+ * edit-distance-1 match there lands on a DIFFERENT word rather than correcting+ * a typo, and because BM25 favors the shorter document the near-miss can+ * outrank the exact hit. Measured: `访问控制` (access control) fell to third+ * behind the non-word `防问控制`, and `검색 품질` (search quality) fell behind+ * `검토` (review).+ *+ * Density, not spacing, is the axis — Hangul is spaced yet behaves like CJK+ * here, while the alphabetic non-Latin scripts (Hebrew, Cyrillic, Greek,+ * Arabic) keep tolerance deliberately, because one edit there is an ordinary+ * typo exactly as in Latin, and each was checked to still rank its exact match+ * first.+ *+ * The non-Hangul ranges mirror `NON_SPACE_SCRIPT_RE` in the app's+ * document-stats, including its escape for the Compatibility Ideographs lower+ * bound: U+F900 NFC-normalizes to U+8C48, so an editor that auto-normalizes+ * source would silently widen the range by ~27K codepoints.+ */+const DENSE_SCRIPT_CHAR =+ /[-ក- -〿-ゟ゠-ヿ㐀-䶿一-鿿\uf900--\u1100-\u11ff\u3130-\u318f\ua960-\ua97f\uac00-\ud7ff]/;+function toleranceFor(intent: WorkspaceSearchIntent, query: string): number {const normalizedQuery = normalize(query);if (normalizedQuery.length < 4) return 0;+ // Orama takes one tolerance for the whole query, so a mixed-script query+ // gives up typo tolerance on its Latin half too — worth it to keep the+ // exact match on top.+ if (DENSE_SCRIPT_CHAR.test(normalizedQuery)) return 0;return intent === 'full_text' ? 1 : 0;}