Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

Preserve JSON when redacting oversized encoded tokens

Ezra Ellettece3409a532fc
diff --git a/apps/screenpipe-app-tauri/lib/trajectories/full-trajectory.test.ts b/apps/screenpipe-app-tauri/lib/trajectories/full-trajectory.test.ts
index 38c95ec22..ccb553686 100644
--- a/apps/screenpipe-app-tauri/lib/trajectories/full-trajectory.test.ts
+++ b/apps/screenpipe-app-tauri/lib/trajectories/full-trajectory.test.ts
@@ -165,6 +165,10 @@ describe("full Workflows trajectory collection", () => {
] });
expect(text).toContain("Keep this text"); expect(text).not.toContain("PRIVATE"); expect(text).not.toContain("base64");
expect([...redactionSegments("a".repeat(100000))]).toEqual(["[OVERSIZED TOKEN REDACTED] "]);
+ // JSON escapes can join otherwise separate lines into one service token.
+ const encoded = contributionText({ text: "word\n".repeat(25000), keep: "Remaining conversation" });
+ expect(JSON.parse(encoded)).toEqual({ text: "[OVERSIZED TOKEN REDACTED]", keep: "Remaining conversation" });
+ expect([...redactionSegments(encoded)].join("")).toBe(encoded);
});
it("preserves upload failure and progress in the normal redacted support evidence", async () => {
browserLogs.reset(); support.write.mockClear();
diff --git a/apps/screenpipe-app-tauri/lib/trajectories/full-trajectory.ts b/apps/screenpipe-app-tauri/lib/trajectories/full-trajectory.ts
index 17d653cde..f310aa3da 100644
--- a/apps/screenpipe-app-tauri/lib/trajectories/full-trajectory.ts
+++ b/apps/screenpipe-app-tauri/lib/trajectories/full-trajectory.ts
@@ -28,16 +28,25 @@ type Ports = {
// Remove recordings and credential fields before even the local outbox write.
// Text and tool arguments/results still pass through strict attested redaction.
export function contributionText(value: unknown): string {
- return JSON.stringify(value, function (key, item) {
+ const text = JSON.stringify(value, function (key, item) {
if (/^(authorization|cookie|password|(?:api|access|refresh|client)?[_-]?(?:key|token|secret))$/i.test(key)) return "[CREDENTIAL OMITTED]";
if (item && typeof item === "object" && /^(image|image_url|input_image|audio|input_audio|video|file)$/i.test(item.type ?? "")) {
return { type: item.type, omitted: "recording" };
}
- if (typeof item === "string") return item.replace(/data:(?:image|audio|video)\/[^;]+;base64,[A-Za-z0-9+/=]+/g, "[RECORDING OMITTED]")
- .replace(/\S+/gu, token => Array.from(token).length > redactionContract.maxInputCharacters - redactionContract.prefix.length
- ? "[OVERSIZED TOKEN REDACTED]" : token);
+ if (typeof item === "string") return item.replace(/data:(?:image|audio|video)\/[^;]+;base64,[A-Za-z0-9+/=]+/g, "[RECORDING OMITTED]");
return item;
}, 2);
+ // Count the encoded token, including escaped tabs/newlines and string/key
+ // punctuation. Replace only string contents so an oversized token never
+ // corrupts JSON and strands the rest of an otherwise valid conversation.
+ const capacity = redactionContract.maxInputCharacters - redactionContract.prefix.length;
+ return text.replace(/"(?:\\.|[^"\\])*"[:,]?/gu, literal => {
+ if (literal.split(" ").every(token => Array.from(token).length <= capacity)) return literal;
+ const closingQuote = literal.lastIndexOf('"');
+ const suffix = literal.slice(closingQuote);
+ return '"' + literal.slice(1, closingQuote).replace(/[^ ]+/gu, token =>
+ Array.from('"' + token + suffix).length > capacity ? "[OVERSIZED TOKEN REDACTED]" : token) + suffix;
+ });
}
// This is the enclave's published 100000-character /filter boundary minus