Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(chat): preserve unicode across context trimming (#6563)

Ezra Ellette5c9a0831b0f7
diff --git a/apps/screenpipe-app-tauri/lib/__tests__/context-pruning.test.ts b/apps/screenpipe-app-tauri/lib/__tests__/context-pruning.test.ts
index cbcc65288..4c8283e50 100644
--- a/apps/screenpipe-app-tauri/lib/__tests__/context-pruning.test.ts
+++ b/apps/screenpipe-app-tauri/lib/__tests__/context-pruning.test.ts
@@ -122,6 +122,31 @@ describe("clampMessageText", () => {
expect(out.endsWith("END")).toBe(true);
expect(out).toContain("trimmed");
});
+
+ it("never splits supplementary code points at history-tail or generic trim boundaries", () => {
+ const historyBefore = "<conversation_history>";
+ const historyAfter = "</conversation_history>\n\nquestion";
+ const historyMarker = "\n…[older history trimmed]\n";
+ const cases = [
+ {
+ text: `${historyBefore}${"old".repeat(30)}😀TAIL${historyAfter}`,
+ maxChars: historyBefore.length + historyMarker.length + 5 + historyAfter.length,
+ },
+ {
+ text: `abc😀${"MIDDLE".repeat(10)}😀xyz`,
+ maxChars: "\n…[trimmed]\n".length + 8,
+ },
+ ];
+
+ for (const { text, maxChars } of cases) {
+ const out = clampMessageText(text, maxChars);
+ expect(out.length).toBeLessThanOrEqual(maxChars);
+ expect([...out].some((char) => {
+ const codePoint = char.codePointAt(0)!;
+ return codePoint >= 0xD800 && codePoint <= 0xDFFF;
+ })).toBe(false);
+ }
+ });
});
// ── pure: boundOversizedMessages ─────────────────────────────────────────
diff --git a/crates/screenpipe-core/assets/extensions/context-pruning.ts b/crates/screenpipe-core/assets/extensions/context-pruning.ts
index cd70eb020..f569701c4 100644
--- a/crates/screenpipe-core/assets/extensions/context-pruning.ts
+++ b/crates/screenpipe-core/assets/extensions/context-pruning.ts
@@ -71,6 +71,16 @@ const CHARS_PER_TOKEN = 4;
const HISTORY_OPEN = "<conversation_history>";
const HISTORY_CLOSE = "</conversation_history>";
+function safeHead(text: string, end: number): string {
+ const last = text.charCodeAt(end - 1);
+ return text.slice(0, last >= 0xD800 && last <= 0xDBFF ? end - 1 : end);
+}
+
+function safeTail(text: string, start: number): string {
+ const first = text.charCodeAt(start);
+ return text.slice(first >= 0xDC00 && first <= 0xDFFF ? start + 1 : start);
+}
+
/** Resolve the model's context window (tokens) from the extension context. */
export function resolveContextWindowTokens(ctx: any): number {
const fromModel = ctx?.model?.contextWindow;
@@ -112,7 +122,7 @@ export function clampMessageText(text: string, maxChars: number): string {
const bodyBudget = maxChars - before.length - after.length - marker.length;
if (bodyBudget > 0) {
// Keep the TAIL of the history body (the most recent turns).
- return before + marker + body.slice(body.length - bodyBudget) + after;
+ return before + marker + safeTail(body, body.length - bodyBudget) + after;
}
// The wrapper + real message already fills the budget: drop the whole
// history body but keep the user's actual message intact.
@@ -122,7 +132,7 @@ export function clampMessageText(text: string, maxChars: number): string {
// Generic oversized payload: keep head + tail so both ends survive.
const marker = "\n…[trimmed]\n";
const half = Math.max(0, Math.floor((maxChars - marker.length) / 2));
- return text.slice(0, half) + marker + text.slice(text.length - half);
+ return safeHead(text, half) + marker + safeTail(text, text.length - half);
}
/**