Skip to content

A personal agent with a browser, terminal, files, and work that keeps going built with CopilotKit and AG-UI.

TypeScript

fix: harden Gmail message parsing (#97)

nvtoan0201-swead989b443ce4
diff --git a/packages/integrations/src/google.ts b/packages/integrations/src/google.ts
index d31d0ce..c94e842 100644
--- a/packages/integrations/src/google.ts
+++ b/packages/integrations/src/google.ts
@@ -164,7 +164,7 @@ function decodeHeader(value: string): string {
),
"latin1",
);
- return new TextDecoder(charset).decode(bytes);
+ return new TextDecoder(charset.split("*")[0]).decode(bytes);
} catch {
return original;
}
@@ -193,7 +193,13 @@ function decodeSnippet(value: string): string {
);
}
function addresses(value: string): string[] {
- return value.match(/[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+/g) ?? [];
+ // A quoted display name may itself contain an e-mail-looking string; only
+ // addresses outside display names count.
+ return (
+ value
+ .replace(/"(?:[^"\\]|\\.)*"\s*(?=<)/g, " ")
+ .match(/[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+/g) ?? []
+ );
}
/** Extract text from a parsed HTML tree. Nothing is rendered or fetched. */
function htmlToPlainText(html: string): string {
@@ -298,12 +304,16 @@ function mapMessage(message: z.infer<typeof messageSchema>): Mail {
const attachments: string[] = [];
const visit = (part: GmailPart, depth: number) => {
if (depth > 30) throw new Error("Gmail message MIME nesting exceeds the limit");
+ // Attachments carry their own content (or an attached message); never merge
+ // their parts into the parent text. A remote body has no filename: it is
+ // hydrated above and still counts as message text.
+ const attached = Boolean(part.filename) || part.mimeType === "message/rfc822";
if (part.filename && part.body?.attachmentId)
attachments.push(
`${message.id}:${part.body.attachmentId}:${encodeURIComponent(wellFormed(part.filename))}`,
);
if (
- !part.filename &&
+ !attached &&
(part.mimeType === "text/plain" || part.mimeType === "text/html") &&
part.body?.data
) {
@@ -315,14 +325,15 @@ function mapMessage(message: z.infer<typeof messageSchema>): Mail {
if (part.mimeType === "text/plain") plain.push(text);
else html.push(htmlToPlainText(text));
}
- for (const child of part.parts ?? []) visit(child, depth + 1);
+ if (!attached) for (const child of part.parts ?? []) visit(child, depth + 1);
};
if (message.payload) visit(message.payload, 0);
const from = decodeHeader(metadata.get("from") ?? "");
const address = addresses(from)[0] ?? from;
- const sender = from.includes("<")
+ const displayName = from.includes("<")
? from.slice(0, from.indexOf("<")).trim().replace(/^"|"$/g, "")
- : address;
+ : "";
+ const sender = displayName || address;
const time = message.internalDate
? Number(message.internalDate)
: Date.parse(metadata.get("date") ?? "");
diff --git a/tests/google.test.ts b/tests/google.test.ts
index b108681..c5e607b 100644
--- a/tests/google.test.ts
+++ b/tests/google.test.ts
@@ -920,6 +920,64 @@ test("single-event validation is repeated at execution and read failures never d
assert.equal(writes, 0);
});
+test("mail parsing tolerates unknown charsets, RFC 2231 words, and display names with addresses", async () => {
+ const client = clientWith((request) =>
+ new URL(request.url).pathname.endsWith("/messages")
+ ? json({ messages: [{ id: "msg-charset" }] })
+ : json({
+ id: "msg-charset",
+ threadId: "thread-charset",
+ payload: {
+ mimeType: "text/plain",
+ headers: [
+ { name: "From", value: "<[email protected]>" },
+ { name: "To", value: '"[email protected]" <[email protected]>' },
+ { name: "Subject", value: "=?UTF-8*en?B?SGVsbG8=?=" },
+ { name: "Content-Type", value: "text/plain; charset=unknown-8bit" },
+ ],
+ body: { data: base64url("Body with unknown charset ✓") },
+ },
+ }),
+ );
+ const [mail] = await client.listMail();
+ assert.equal(mail.body, "Body with unknown charset ✓");
+ assert.equal(mail.subject, "Hello");
+ assert.equal(mail.from, "[email protected]");
+ assert.equal(mail.sender, "[email protected]");
+ assert.deepEqual(mail.to, ["[email protected]"]);
+});
+
+test("an attached message is not merged into the parent body", async () => {
+ const client = clientWith((request) =>
+ new URL(request.url).pathname.endsWith("/messages")
+ ? json({ messages: [{ id: "msg-forward" }] })
+ : json({
+ id: "msg-forward",
+ threadId: "thread-forward",
+ payload: {
+ mimeType: "multipart/mixed",
+ parts: [
+ { mimeType: "text/plain", body: { data: base64url("Outer body text.") } },
+ {
+ mimeType: "message/rfc822",
+ filename: "forwarded.eml",
+ body: { attachmentId: "attach2", size: 42 },
+ parts: [
+ {
+ mimeType: "text/plain",
+ body: { data: base64url("INNER ATTACHED MESSAGE BODY") },
+ },
+ ],
+ },
+ ],
+ },
+ }),
+ );
+ const [mail] = await client.listMail();
+ assert.equal(mail.body, "Outer body text.");
+ assert.deepEqual(mail.attachments, ["msg-forward:attach2:forwarded.eml"]);
+});
+
test("a sender's unpaired-surrogate attachment filename does not break the inbox", async () => {
const client = clientWith((request) => {
const url = new URL(request.url);