Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): index non-Latin scripts in workspace search (#3322)

mike-inkeep2a6cbe5ac517
diff --git a/.changeset/non-latin-search-tokenizer.md b/.changeset/non-latin-search-tokenizer.md
new file mode 100644
index 000000000..86a603096
--- /dev/null
+++ b/.changeset/non-latin-search-tokenizer.md
@@ -0,0 +1,5 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+Workspace search now finds content written in non-Latin scripts. Previously the search index kept only Latin-script words, so document bodies in Hebrew, Cyrillic, Greek, Arabic, Thai, Chinese, Japanese, Korean, and other scripts indexed to zero searchable words — the omnibar and the MCP `search` tool couldn't find them at all, and in a mixed-script query only the Latin words did any work. Search now additionally segments those scripts with the runtime's built-in word segmenter (dictionary-based for unspaced scripts like Thai, Japanese, and Chinese), with no new dependencies and no change to how Latin-only content is matched or ranked. In scripts where a single character carries a whole syllable — Chinese, Japanese, Korean, Thai, Khmer — queries are matched exactly rather than with the one-character typo tolerance applied elsewhere, since there a one-character difference is a different word and near-misses would crowd out the exact match. Thanks to @Guohao1020 for the report, root-cause analysis, and proposed approach, and to @AsafDafna for the implementation and validation (#903).
diff --git a/packages/core/src/search/workspace-search.non-latin.test.ts b/packages/core/src/search/workspace-search.non-latin.test.ts
new file mode 100644
index 000000000..317e31314
--- /dev/null
+++ b/packages/core/src/search/workspace-search.non-latin.test.ts
@@ -0,0 +1,263 @@
+import { describe, expect, test, vi } from 'vitest';
+import {
+ createWorkspaceSearchCorpus,
+ createWorkspaceSearchDocument,
+ searchWorkspaceCorpus,
+} from './workspace-search.ts';
+
+/**
+ * Non-Latin body search (inkeep/open-knowledge#903).
+ *
+ * Orama's default `english` tokenizer treats every character outside its Latin
+ * class as a separator, so non-Latin body content indexed to ZERO tokens and
+ * was unsearchable — masked for file NAMES by the lexical ladder, which
+ * compares whole strings. These tests search by BODY content behind
+ * Latin-named docs, so a hit can only come from the BM25 index and a
+ * tokenizer regression cannot hide behind the name ladder.
+ */
+
+const docs = [
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/coal.md',
+ title: 'coal notes',
+ content: 'הפקת פחם באירופה במאה התשע עשרה',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/access.md',
+ title: 'access notes',
+ content: '访问控制与权限管理的最佳实践',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/tokyo.md',
+ title: 'tokyo notes',
+ content: '検索テキストの品質を確認します',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/tenants.md',
+ title: 'tenant notes',
+ content: 'поиск работает: tenant isolation details',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/thai.md',
+ title: 'thai notes',
+ content: 'ค้นหาข้อความภาษาไทย',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/korean.md',
+ title: 'korean notes',
+ content: '검색 품질을 확인합니다',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/avalanche.md',
+ title: 'avalanche notes',
+ content: 'çığ düştü ve yol kapandı',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/plain.md',
+ title: 'plain notes',
+ content: 'a plain english document about foxes',
+ }),
+];
+
+const corpus = createWorkspaceSearchCorpus(docs);
+
+function pathsFor(query: string): string[] {
+ return searchWorkspaceCorpus(corpus, query, { intent: 'full_text' }).map(
+ (result) => result.document.path,
+ );
+}
+
+describe('non-Latin body content is searchable', () => {
+ test.each([
+ ['Hebrew', 'פחם', 'notes/coal.md'],
+ ['Chinese', '访问控制', 'notes/access.md'],
+ ['Japanese', '検索テキスト', 'notes/tokyo.md'],
+ ['Cyrillic', 'поиск работает', 'notes/tenants.md'],
+ ['Thai', 'ค้นหา', 'notes/thai.md'],
+ ['Korean', '검색', 'notes/korean.md'],
+ ])('%s query finds body content', (_script, query, expectedPath) => {
+ expect(pathsFor(query)).toContain(expectedPath);
+ });
+
+ test('accented Latin outside the default set is searchable', () => {
+ // Latin-script letters outside the splitter's kept set (`àèéìòóù`) are
+ // separators too. `çığ` is made only of those, so the default tokenizer
+ // reduces it to ZERO tokens on both sides and it is unfindable — the same
+ // failure as the non-Latin scripts above, in Latin script. Words that keep
+ // some fragment (`niño` → `ni`/`o`) stay findable either way, so this
+ // all-dropped case is what actually pins the behavior.
+ expect(pathsFor('çığ')).toContain('notes/avalanche.md');
+ });
+
+ test('mixed-script query uses both halves', () => {
+ // The Cyrillic half alone must be able to select the doc: `поиск` appears
+ // only in tenants.md, so a tokenizer that silently drops the non-Latin
+ // half would still pass a combined-terms assertion — pin the ranking too.
+ const paths = pathsFor('поиск isolation');
+ expect(paths[0]).toBe('notes/tenants.md');
+ expect(pathsFor('поиск')).toContain('notes/tenants.md');
+ });
+
+ test('the omnibar call shape (full_text candidates, navigation ranking) finds non-Latin content', () => {
+ const results = searchWorkspaceCorpus(corpus, '访问控制', {
+ intent: 'full_text',
+ ranking: 'navigation',
+ });
+ expect(results.map((result) => result.document.path)).toContain('notes/access.md');
+ });
+
+ test('pure-Latin queries are unaffected', () => {
+ expect(pathsFor('foxes')[0]).toBe('notes/plain.md');
+ });
+});
+
+describe('dense scripts do not take edit-distance tolerance', () => {
+ // Now that these scripts tokenize at all, the `full_text` tolerance of 1
+ // reaches them for the first time. A word there is a few characters long, so
+ // one edit is a different word, not a typo — and BM25 can float that
+ // near-miss above the exact hit.
+ const cjk = createWorkspaceSearchCorpus([
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/access.md',
+ title: 'access notes',
+ content: '访问控制与权限管理',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/variant.md',
+ title: 'variant notes',
+ // `防问控制` — one character off, and not a word.
+ content: '防问控制与权限管理',
+ }),
+ // A genuinely related doc. It legitimately ranks first (shorter, and it
+ // really does contain 访问 + 控制), and its presence is what shifts the
+ // corpus statistics far enough for the near-miss to beat the exact hit.
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/controller.md',
+ title: 'controller notes',
+ content: '访问控制器的配置说明',
+ }),
+ ]);
+
+ test('a one-character variant does not outrank the exact match', () => {
+ const paths = searchWorkspaceCorpus(cjk, '访问控制', { intent: 'full_text' }).map(
+ (result) => result.document.path,
+ );
+ expect(paths.indexOf('notes/access.md')).toBeLessThan(paths.indexOf('notes/variant.md'));
+ });
+
+ // Hangul is space-separated, so it is NOT an unspaced script — but a syllable
+ // block carries as much as a whole Latin word, so it fails the same way:
+ // `검색` (search) vs `검토` (review) vs `검사` (inspection) are pairwise one
+ // edit apart and mean different things.
+ const hangul = createWorkspaceSearchCorpus([
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/search.md',
+ title: 'search notes',
+ content: '검색 품질을 확인합니다',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/review.md',
+ title: 'review notes',
+ content: '검토 품질을 확인합니다',
+ }),
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/inspection.md',
+ title: 'inspection notes',
+ content: '검사 결과를 확인합니다',
+ }),
+ ]);
+
+ test('a one-syllable Hangul variant does not outrank the exact match', () => {
+ const paths = searchWorkspaceCorpus(hangul, '검색 품질', { intent: 'full_text' }).map(
+ (result) => result.document.path,
+ );
+ expect(paths[0]).toBe('notes/search.md');
+ });
+
+ // The other half of the contract: alphabetic non-Latin scripts must KEEP
+ // tolerance, so widening the dense-script ranges over them is caught. The
+ // recall tests above all query exact terms and would survive that widening.
+ const cyrillic = createWorkspaceSearchCorpus([
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/russian.md',
+ title: 'russian notes',
+ content: 'поиск работает хорошо',
+ }),
+ ]);
+
+ test('a one-letter Cyrillic typo still finds the document', () => {
+ // `поиек` (е for с) appears nowhere in the corpus — only tolerance finds it.
+ expect(
+ searchWorkspaceCorpus(cyrillic, 'поиек', { intent: 'full_text' }).map(
+ (result) => result.document.path,
+ ),
+ ).toContain('notes/russian.md');
+ });
+});
+
+describe('input the base tokenizer already handles stays untouched', () => {
+ function topScoreFor(content: string): number {
+ const single = createWorkspaceSearchCorpus([
+ createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/school.md',
+ title: 'school notes',
+ content,
+ }),
+ ]);
+ const results = searchWorkspaceCorpus(single, 'ecole', { intent: 'full_text' });
+ expect(results).not.toHaveLength(0);
+ return results[0].score;
+ }
+
+ test('capitalizing an accented word does not change its score', () => {
+ // Capitalization is not a script difference: the base lowercases before it
+ // splits, so `École` and `école` yield the same base tokens (`ecole`,
+ // diacritic-folded). Treating `É` as dropped would append a second,
+ // unfolded `école` to the capitalized doc only — same hits, but a longer
+ // field and therefore a different BM25 score.
+ expect(topScoreFor('École normale')).toBe(topScoreFor('école normale'));
+ });
+
+ test('a runtime without Intl.Segmenter degrades instead of failing module load', async () => {
+ // This module is imported by the command palette and the wiki-link
+ // suggestion extension, so an unguarded constructor would take down the
+ // app bundle's module evaluation, not just non-Latin search.
+ const descriptor = Object.getOwnPropertyDescriptor(Intl, 'Segmenter');
+ Reflect.deleteProperty(Intl, 'Segmenter');
+ vi.resetModules();
+ try {
+ const mod = await import('./workspace-search.ts');
+ const fallbackCorpus = mod.createWorkspaceSearchCorpus([
+ mod.createWorkspaceSearchDocument({
+ kind: 'page',
+ path: 'notes/plain.md',
+ title: 'plain notes',
+ content: 'a plain english document about foxes',
+ }),
+ ]);
+ const paths = mod
+ .searchWorkspaceCorpus(fallbackCorpus, 'foxes', { intent: 'full_text' })
+ .map((result) => result.document.path);
+ expect(paths).toContain('notes/plain.md');
+ } finally {
+ if (descriptor) Object.defineProperty(Intl, 'Segmenter', descriptor);
+ vi.resetModules();
+ }
+ });
+});
diff --git a/packages/core/src/search/workspace-search.ts b/packages/core/src/search/workspace-search.ts
index f03433a16..a9072338a 100644
--- a/packages/core/src/search/workspace-search.ts
+++ b/packages/core/src/search/workspace-search.ts
@@ -1,4 +1,5 @@
import { type AnyOrama, count, create, insertMultiple, removeMultiple, search } from '@orama/orama';
+import { tokenizer as oramaTokenizer } from '@orama/orama/components';
import { isHiddenDocName } from '../util/doc-name.ts';
export type WorkspaceSearchKind = 'page' | 'folder' | 'file';
@@ -253,10 +254,96 @@ export function createWorkspaceSearchDocument(input: {
};
}
+/**
+ * Matches any letter or number that Orama's default `english` tokenizer treats
+ * as a separator rather than a word character. The default splitter
+ * (`/[^A-Za-zàèéìòóù0-9_'-]+/`) keeps only that Latin class, so a run of
+ * Hebrew, Cyrillic, Greek, Arabic, CJK, Thai, … is deleted wholesale on both
+ * the indexing and the query side — such content indexes to zero tokens and is
+ * unsearchable. Read the double negation as: a Unicode letter NOT in the
+ * default-kept Latin set, or a Unicode number NOT in `0-9`.
+ *
+ * Case-insensitive because the base splits `input.toLowerCase()`, so the kept
+ * set is really a set of case-folded letters: `É` is kept (it folds to the kept
+ * `é`) and must NOT count as dropped, or `École` would index the base's
+ * diacritic-folded `ecole` AND an appended `école`, inflating `fieldLength`
+ * for input the base handles fine. Folding also covers lookalikes that fold
+ * into the set, such as the Kelvin sign `K` → `k`.
+ *
+ * The number half admits only `Nd` and `Nl` — the number categories whose
+ * characters segment as word-like. `No` (`½`, `²`, `¾`) never does, so
+ * admitting it would route a whole document through the segmenter to append
+ * nothing: a single `m²` used to double that document's index-build cost
+ * (2000 Latin docs / 1.5 MB: 93 ms admitting `No`, 46 ms not, same tokens).
+ *
+ * Cost when the gate does fire is the segmentation itself, on the same event
+ * loop as the index build: for 2000 docs it is 91 ms Hebrew (3.4 MB), 140 ms
+ * Thai (6.0 MB), and 163 ms Chinese (3.0 MB), against 78 ms for a Latin corpus
+ * that never reaches the segmenter.
+ */
+const DROPPED_SCRIPT_CHAR = /[^\P{L}a-zàèéìòóù]|[^\P{Nd}0-9]|\p{Nl}/iu;
+
+/**
+ * Word segmenter for scripts the default tokenizer drops: plain word splitting
+ * for spaced scripts (Hebrew, Cyrillic, Greek, Arabic), dictionary-based
+ * segmentation for unspaced ones (Thai, Japanese, Chinese). Ships with every
+ * supported runtime (Node ≥ 16, Electron, evergreen browsers) — no dependency.
+ * The locale is pinned so token output never varies with the host machine's
+ * default locale: word-granularity segmentation dispatches by the script of
+ * the text itself, so the pinned value costs nothing for non-English content,
+ * and index/query tokenization stays deterministic across environments (and in
+ * token-level tests). Stateless, so one shared instance serves every index.
+ *
+ * Feature-detected rather than constructed outright: this module is imported by
+ * the command palette and the wiki-link suggestion extension, so a throw here
+ * would fail the app bundle's module evaluation on a runtime without the API,
+ * not merely degrade non-Latin search. `null` falls back to base tokenization,
+ * i.e. Latin-only search.
+ */
+const WORD_SEGMENTER = (() => {
+ const Ctor = (globalThis as { Intl: { Segmenter?: typeof Intl.Segmenter } }).Intl.Segmenter;
+ return Ctor ? new Ctor('en', { granularity: 'word' }) : null;
+})();
+
+/**
+ * The default Orama tokenizer, extended to ALSO emit tokens for scripts the
+ * default splitter deletes. Latin-only input (no {@link DROPPED_SCRIPT_CHAR})
+ * returns the base result unchanged — same tokens, same order — so existing
+ * tokenization and BM25 scores are byte-identical and the pinned flag-OFF
+ * parity fixture keeps passing. Input containing a dropped script additionally
+ * yields that script's `Intl.Segmenter` word segments, lowercased to match the
+ * base pipeline and deduplicated like the base's `allowDuplicates: false`.
+ * Orama routes both indexing and querying through this tokenizer, so the two
+ * sides stay consistent.
+ */
+function createWorkspaceSearchTokenizer() {
+ const base = oramaTokenizer.createTokenizer({ language: 'english' });
+ const baseTokenize = base.tokenize.bind(base);
+ base.tokenize = (input, language, prop, withCache) => {
+ const tokens = baseTokenize(input, language, prop, withCache);
+ if (!WORD_SEGMENTER || typeof input !== 'string') return tokens;
+ if (!DROPPED_SCRIPT_CHAR.test(input)) return tokens;
+ const seen = new Set(tokens);
+ for (const { segment, isWordLike } of WORD_SEGMENTER.segment(input)) {
+ if (!isWordLike || !DROPPED_SCRIPT_CHAR.test(segment)) continue;
+ const token = segment.toLowerCase();
+ if (!seen.has(token)) {
+ seen.add(token);
+ tokens.push(token);
+ }
+ }
+ return tokens;
+ };
+ return base;
+}
+
function createWorkspaceSearchIndex(
documents: readonly WorkspaceSearchDocument[],
): WorkspaceSearchIndex {
- const db = create({ schema: WORKSPACE_SEARCH_SCHEMA });
+ const db = create({
+ schema: WORKSPACE_SEARCH_SCHEMA,
+ components: { tokenizer: createWorkspaceSearchTokenizer() },
+ });
if (documents.length > 0) {
insertMultiple(db, documents as WorkspaceSearchDocument[]);
}
@@ -663,9 +750,36 @@ function searchBoost(
return { title: 8, name: 7, path: 5, pathSegments: 4 };
}
+/**
+ * Scripts where one character carries a whole syllable or morpheme, so a word
+ * is only a few characters long: CJK, Thai, Khmer, and Hangul. An
+ * edit-distance-1 match there lands on a DIFFERENT word rather than correcting
+ * a typo, and because BM25 favors the shorter document the near-miss can
+ * outrank the exact hit. Measured: `访问控制` (access control) fell to third
+ * behind the non-word `防问控制`, and `검색 품질` (search quality) fell behind
+ * `검토` (review).
+ *
+ * Density, not spacing, is the axis — Hangul is spaced yet behaves like CJK
+ * here, while the alphabetic non-Latin scripts (Hebrew, Cyrillic, Greek,
+ * Arabic) keep tolerance deliberately, because one edit there is an ordinary
+ * typo exactly as in Latin, and each was checked to still rank its exact match
+ * first.
+ *
+ * The non-Hangul ranges mirror `NON_SPACE_SCRIPT_RE` in the app's
+ * document-stats, including its escape for the Compatibility Ideographs lower
+ * bound: U+F900 NFC-normalizes to U+8C48, so an editor that auto-normalizes
+ * source would silently widen the range by ~27K codepoints.
+ */
+const DENSE_SCRIPT_CHAR =
+ /[฀-๿ក-៿ -〿぀-ゟ゠-ヿ㐀-䶿一-鿿\uf900-﫿＀-￯\u1100-\u11ff\u3130-\u318f\ua960-\ua97f\uac00-\ud7ff]/;
+
function toleranceFor(intent: WorkspaceSearchIntent, query: string): number {
const normalizedQuery = normalize(query);
if (normalizedQuery.length < 4) return 0;
+ // Orama takes one tolerance for the whole query, so a mixed-script query
+ // gives up typo tolerance on its Latin half too — worth it to keep the
+ // exact match on top.
+ if (DENSE_SCRIPT_CHAR.test(normalizedQuery)) return 0;
return intent === 'full_text' ? 1 : 0;
}