LLM Wiki is a cross-platform desktop application that turns your documents into an organized, interlinked knowledge base — automatically. Instead of traditional RAG (retrieve-and-answer from scratch every time), the LLM incrementally builds and maintains a persistent wiki from your sources。
TypeScript
fix: Chinese search now works — CJK text split into bigrams + chars
nash_su81d0dbd913ca
diff --git a/src/lib/search.ts b/src/lib/search.tsindex 68b7fd55..5c97883f 100644--- a/src/lib/search.ts+++ b/src/lib/search.ts@@ -22,11 +22,42 @@ const STOP_WORDS = new Set([])export function tokenizeQuery(query: string): string[] {- return query+ // Split by whitespace and punctuation+ const rawTokens = query.toLowerCase()- .split(/[\s,,。!?、;:""''()()\-_/\\]+/)+ .split(/[\s,,。!?、;:""''()()\-_/\\·~~…]+/).filter((t) => t.length > 1).filter((t) => !STOP_WORDS.has(t))++ const tokens: string[] = []++ for (const token of rawTokens) {+ // Check if token contains CJK characters+ const hasCJK = /[\u4e00-\u9fff\u3400-\u4dbf]/.test(token)++ if (hasCJK && token.length > 2) {+ // For CJK text: split into individual characters AND overlapping bigrams+ // "默会知识" → ["默会", "会知", "知识", "默", "会", "知", "识"]+ const chars = [...token]+ // Add bigrams (most useful for Chinese)+ for (let i = 0; i < chars.length - 1; i++) {+ tokens.push(chars[i] + chars[i + 1])+ }+ // Also add individual chars (for single-char matches)+ for (const ch of chars) {+ if (!STOP_WORDS.has(ch)) {+ tokens.push(ch)+ }+ }+ // Keep the original token too (for exact phrase match)+ tokens.push(token)+ } else {+ tokens.push(token)+ }+ }++ // Deduplicate+ return [...new Set(tokens)]}function tokenMatchScore(text: string, tokens: readonly string[]): number {