Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
perf(desktop): accelerate long ASCII search snippets
ficowang053f7779f596
diff --git a/apps/desktop/src/main/localDb/__tests__/cjkSeg.test.ts b/apps/desktop/src/main/localDb/__tests__/cjkSeg.test.tsindex 0d6d9c53fa..589d18d106 100644--- a/apps/desktop/src/main/localDb/__tests__/cjkSeg.test.ts+++ b/apps/desktop/src/main/localDb/__tests__/cjkSeg.test.ts@@ -111,6 +111,28 @@ describe('buildSnippetFromContent', () => {expect(snippet.endsWith('…')).toBe(true);});+ it('ASCII 快路保持重复命中、实体转义和 16384 码点窗口边界', () => {+ expect(buildSnippetFromContent('LOGIN login LoGiN & <tag>', ['login'])).toBe(+ '<mark>LOGIN</mark> <mark>login</mark> <mark>LoGiN</mark> & <tag>',+ );++ const atLastWindowPosition = `${'a'.repeat(SNIPPET_SOURCE_MAX_CHARS - 6)}Needleafter`;+ expect(buildSnippetFromContent(atLastWindowPosition, ['needle'])).toBe(+ `…${'a'.repeat(24)}<mark>Needle</mark>`,+ );+ const justOutsideWindow = `${'a'.repeat(SNIPPET_SOURCE_MAX_CHARS)}needle`;+ expect(buildSnippetFromContent(justOutsideWindow, ['needle'])).toBe(`${'a'.repeat(24)}…`);+ });++ it('Unicode 慢路保持 combining mark、emoji 和孤立 surrogate 的码点坐标', () => {+ expect(buildSnippetFromContent('😀甲\u0301乙 e\u0301', ['甲\u0301乙', 'e\u0301'])).toBe(+ '😀<mark>甲́乙</mark> <mark>é</mark>',+ );+ expect(buildSnippetFromContent('😀\ud800needle\udc00', ['needle'])).toBe(+ '😀\ud800<mark>needle</mark>\udc00',+ );+ });+it('真实 FTS 链路:snippet 不篡改中英相邻与原文真实空格', () => {const db = new Database(':memory:');registerCjkSeg(db);diff --git a/apps/desktop/src/main/localDb/cjkSeg.ts b/apps/desktop/src/main/localDb/cjkSeg.tsindex 14f6a5e814..c8d1af000b 100644--- a/apps/desktop/src/main/localDb/cjkSeg.ts+++ b/apps/desktop/src/main/localDb/cjkSeg.ts@@ -155,36 +155,39 @@ export function buildSnippetFromContent(content: unknown, queryTokens: readonlyif (!raw) return '';const text = takeCodePoints(raw, SNIPPET_SOURCE_MAX_CHARS);- const aligned = cjkSegAligned(text);- if (!aligned || aligned.segChars.length === 0) return fallbackSnippet(text);- const { segChars, segToOrig } = aligned;- const foldedSeg = segChars.map((ch) => ch.toLowerCase());-// 把每个查询 token 的分词形态在码点数组里定位成 [segStart, segEnd) 区间,// 再经 segToOrig 映回原文码点区间(跳过 -1 的插入空格位)。// 查找全程用码点下标,不能用 String.indexOf / .length(那是 UTF-16 单元)。- const spans: Array<{ start: number; end: number }> = [];- for (const token of queryTokens) {- const needle = cjkSeg(token);- if (!needle || needle.length === 0) continue;- const needleChars = [...needle];- const foldedNeedle = needleChars.map((ch) => ch.toLowerCase());- let from = 0;- for (;;) {- const at = indexOfCodePoints(foldedSeg, foldedNeedle, from);- if (at < 0) break;- from = at + needleChars.length;- const first = segToOrig[at]!;- const last = segToOrig[at + needleChars.length - 1]!;- if (first < 0 || last < 0) continue;- const start = Math.min(first, last);- const end = Math.max(first, last) + 1;- spans.push({ start, end });+ const asciiSpans = findAsciiSnippetSpans(text, queryTokens);+ const spans: Array<{ start: number; end: number }> = asciiSpans ?? [];+ if (!asciiSpans) {+ const aligned = cjkSegAligned(text);+ if (!aligned || aligned.segChars.length === 0) return fallbackSnippet(text);+ const { segChars, segToOrig } = aligned;+ const foldedSeg = segChars.map((ch) => ch.toLowerCase());++ for (const token of queryTokens) {+ const needle = cjkSeg(token);+ if (!needle || needle.length === 0) continue;+ const needleChars = [...needle];+ const foldedNeedle = needleChars.map((ch) => ch.toLowerCase());+ let from = 0;+ for (;;) {+ const at = indexOfCodePoints(foldedSeg, foldedNeedle, from);+ if (at < 0) break;+ from = at + needleChars.length;+ const first = segToOrig[at]!;+ const last = segToOrig[at + needleChars.length - 1]!;+ if (first < 0 || last < 0) continue;+ const start = Math.min(first, last);+ const end = Math.max(first, last) + 1;+ spans.push({ start, end });+ }}}- const origChars = [...text];if (spans.length === 0) return fallbackSnippet(text);+ const origChars = [...text];spans.sort((a, b) => a.start - b.start || a.end - b.end);const merged: Array<{ start: number; end: number }> = [];@@ -224,12 +227,47 @@ export function buildSnippetFromContent(content: unknown, queryTokens: readonlyreturn out;function fallbackSnippet(t: string): string {- const chars = [...t];+ const chars = [...takeCodePoints(t, SNIPPET_CONTEXT_CHARS * 2 + 1)];if (chars.length <= SNIPPET_CONTEXT_CHARS * 2) return escapeSnippetText(t);return `${escapeSnippetText(chars.slice(0, SNIPPET_CONTEXT_CHARS).join(''))}…`;}}+/** ASCII 码点与 UTF-16 单元一一对应,可以直接使用原生字符串搜索而不改变坐标。 */+function findAsciiSnippetSpans(+ text: string,+ queryTokens: readonly string[],+): Array<{ start: number; end: number }> | null {+ if (+ !isAscii(text) ||+ !queryTokens.every((token) => typeof token === 'string' && isAscii(token))+ ) {+ return null;+ }+ const foldedText = text.toLowerCase();+ const spans: Array<{ start: number; end: number }> = [];+ for (const token of queryTokens) {+ if (token.length === 0) continue;+ const foldedNeedle = token.toLowerCase();+ let from = 0;+ for (;;) {+ const at = foldedText.indexOf(foldedNeedle, from);+ if (at < 0) break;+ const end = at + token.length;+ spans.push({ start: at, end });+ from = end;+ }+ }+ return spans;+}++function isAscii(text: string): boolean {+ for (let index = 0; index < text.length; index += 1) {+ if (text.charCodeAt(index) > 0x7f) return false;+ }+ return true;+}+/*** snippet 走 `<mark>` 哨兵协议。原文里的 `<` / `>` 必须先转成实体,* 否则用户内容里的真 `<mark>` / `</mark>` 会被消费端当成控制标记,@@ -243,10 +281,13 @@ function escapeSnippetText(text: string): string {}function takeCodePoints(text: string, max: number): string {- const chars: string[] = [];+ if (text.length <= max) return text;+ let codePoints = 0;+ let codeUnits = 0;for (const ch of text) {- if (chars.length >= max) return chars.join('');- chars.push(ch);+ if (codePoints >= max) return text.slice(0, codeUnits);+ codePoints += 1;+ codeUnits += ch.length;}return text;}