YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
fix(chat): preserve unicode across context trimming (#6563)
Ezra Ellette5c9a0831b0f7
diff --git a/apps/screenpipe-app-tauri/lib/__tests__/context-pruning.test.ts b/apps/screenpipe-app-tauri/lib/__tests__/context-pruning.test.tsindex cbcc65288..4c8283e50 100644--- a/apps/screenpipe-app-tauri/lib/__tests__/context-pruning.test.ts+++ b/apps/screenpipe-app-tauri/lib/__tests__/context-pruning.test.ts@@ -122,6 +122,31 @@ describe("clampMessageText", () => {expect(out.endsWith("END")).toBe(true);expect(out).toContain("trimmed");});++ it("never splits supplementary code points at history-tail or generic trim boundaries", () => {+ const historyBefore = "<conversation_history>";+ const historyAfter = "</conversation_history>\n\nquestion";+ const historyMarker = "\n…[older history trimmed]\n";+ const cases = [+ {+ text: `${historyBefore}${"old".repeat(30)}😀TAIL${historyAfter}`,+ maxChars: historyBefore.length + historyMarker.length + 5 + historyAfter.length,+ },+ {+ text: `abc😀${"MIDDLE".repeat(10)}😀xyz`,+ maxChars: "\n…[trimmed]\n".length + 8,+ },+ ];++ for (const { text, maxChars } of cases) {+ const out = clampMessageText(text, maxChars);+ expect(out.length).toBeLessThanOrEqual(maxChars);+ expect([...out].some((char) => {+ const codePoint = char.codePointAt(0)!;+ return codePoint >= 0xD800 && codePoint <= 0xDFFF;+ })).toBe(false);+ }+ });});// ── pure: boundOversizedMessages ─────────────────────────────────────────diff --git a/crates/screenpipe-core/assets/extensions/context-pruning.ts b/crates/screenpipe-core/assets/extensions/context-pruning.tsindex cd70eb020..f569701c4 100644--- a/crates/screenpipe-core/assets/extensions/context-pruning.ts+++ b/crates/screenpipe-core/assets/extensions/context-pruning.ts@@ -71,6 +71,16 @@ const CHARS_PER_TOKEN = 4;const HISTORY_OPEN = "<conversation_history>";const HISTORY_CLOSE = "</conversation_history>";+function safeHead(text: string, end: number): string {+ const last = text.charCodeAt(end - 1);+ return text.slice(0, last >= 0xD800 && last <= 0xDBFF ? end - 1 : end);+}++function safeTail(text: string, start: number): string {+ const first = text.charCodeAt(start);+ return text.slice(first >= 0xDC00 && first <= 0xDFFF ? start + 1 : start);+}+/** Resolve the model's context window (tokens) from the extension context. */export function resolveContextWindowTokens(ctx: any): number {const fromModel = ctx?.model?.contextWindow;@@ -112,7 +122,7 @@ export function clampMessageText(text: string, maxChars: number): string {const bodyBudget = maxChars - before.length - after.length - marker.length;if (bodyBudget > 0) {// Keep the TAIL of the history body (the most recent turns).- return before + marker + body.slice(body.length - bodyBudget) + after;+ return before + marker + safeTail(body, body.length - bodyBudget) + after;}// The wrapper + real message already fills the budget: drop the whole// history body but keep the user's actual message intact.@@ -122,7 +132,7 @@ export function clampMessageText(text: string, maxChars: number): string {// Generic oversized payload: keep head + tail so both ends survive.const marker = "\n…[trimmed]\n";const half = Math.max(0, Math.floor((maxChars - marker.length) / 2));- return text.slice(0, half) + marker + text.slice(text.length - half);+ return safeHead(text, half) + marker + safeTail(text, text.length - half);}/**