Skip to content

A personal agent with a browser, terminal, files, and work that keeps going built with CopilotKit and AG-UI.

TypeScript

fix: decode mail bodies with an unknown charset as UTF-8 instead of failing the list (#141)

kevineeeb4edfb19b
diff --git a/packages/integrations/src/google.ts b/packages/integrations/src/google.ts
index 63ea131..7a6ae71 100644
--- a/packages/integrations/src/google.ts
+++ b/packages/integrations/src/google.ts
@@ -270,6 +270,14 @@ function htmlToPlainText(html: string): string {
.replace(/\n{3,}/g, "\n\n")
.trim();
}
+/** UTF-8 when the declared charset is not one TextDecoder knows (`unknown-8bit`, `utf-7`). */
+function decoderFor(charset: string): TextDecoder {
+ try {
+ return new TextDecoder(charset);
+ } catch {
+ return new TextDecoder();
+ }
+}
/**
* Replace unpaired surrogates with U+FFFD so encodeURIComponent cannot throw "URI
* malformed" on a sender-supplied name. (String.prototype.toWellFormed, but the
@@ -301,7 +309,7 @@ function mapMessage(message: z.infer<typeof messageSchema>): Mail {
headers(part)
.get("content-type")
?.match(/charset=["']?([^;"'\s]+)/i)?.[1] ?? "utf-8";
- const text = new TextDecoder(charset).decode(decodeBase64url(part.body.data, 1024 * 1024));
+ const text = decoderFor(charset).decode(decodeBase64url(part.body.data, 1024 * 1024));
if (part.mimeType === "text/plain") plain.push(text);
else html.push(htmlToPlainText(text));
}
diff --git a/tests/google.test.ts b/tests/google.test.ts
index 47b5990..c5a2535 100644
--- a/tests/google.test.ts
+++ b/tests/google.test.ts
@@ -88,6 +88,35 @@ test("mail reads nested plain text and attachment references over authenticated
assert.deepEqual(mail.attachments, ["msg1:attach1:Permission%3A%20form.pdf"]);
});
+test("a body charset TextDecoder does not know falls back to UTF-8 instead of failing the list", async () => {
+ const charsets: Record<string, string> = { known: "utf-8", unknown: "unknown-8bit" };
+ const bodies: Record<string, string> = { known: "Opening hours", unknown: "Café tickets" };
+ const client = clientWith((request) => {
+ const url = new URL(request.url);
+ if (url.pathname.endsWith("/messages"))
+ return json({ messages: [{ id: "known" }, { id: "unknown" }] });
+ const id = url.pathname.split("/").at(-1) ?? "";
+ return json({
+ id,
+ threadId: `thread-${id}`,
+ labelIds: ["INBOX"],
+ payload: {
+ headers: [
+ { name: "From", value: "Museum <[email protected]>" },
+ { name: "Content-Type", value: `text/plain; charset=${charsets[id]}` },
+ ],
+ mimeType: "text/plain",
+ body: { data: base64url(bodies[id]) },
+ },
+ });
+ });
+ const mail = await client.listMail();
+ assert.deepEqual(
+ mail.map((item) => item.body),
+ ["Opening hours", "Café tickets"],
+ );
+});
+
test("HTML-only messages expose complete plain text while removing active and non-content elements", async () => {
let reads = 0;
const fullText = "Complete museum itinerary. ".repeat(4500);