Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.
Decode an inbound email part in the charset it names, not always as UTF-8 (#787)
kevincf426553f10e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 346e71fc..f1244b65 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -38,6 +38,12 @@ An email trigger reads an HTML-only mail by stripping its tags, and the charactethe Bot read `AT&T`, `we’re` and `Q3 numbers`. Numeric references, `&`, `<`,`>`, `"`, `'` and ` ` are now decoded, once, so `&lt;` reads as `<`.+### An email in another charset reaches its trigger as the sender wrote it++A base64 or quoted-printable email part was always read as UTF-8, whatever charset its Content-Type+named, so an `iso-8859-1` mail reached the Bot with every accented letter turned into `�`. Each part+is now decoded in the charset it names, and a charset the runtime does not know still reads as UTF-8.+### The Bot's screen opens only when there is something to watchAsking a Bot a question it answered in words could still open its computer beside the conversation,diff --git a/server/src/responsibilities/email.ts b/server/src/responsibilities/email.tsindex e65d674d..f477e639 100644--- a/server/src/responsibilities/email.ts+++ b/server/src/responsibilities/email.ts@@ -157,19 +157,46 @@ function unfoldHeaders(block: string) {}return headers;}-function decodeBody(body: string, encoding: string | undefined) {+/**+ * The text a part's transfer-encoded bytes spell, in the charset its Content-Type names.+ *+ * Base64 and quoted-printable carry bytes, not text: a `charset=iso-8859-1` part saying "caf=E9" is+ * "café", and reading those bytes as UTF-8 turned every accented letter into U+FFFD. A charset the+ * runtime does not know falls back to UTF-8, as before.+ */+function decodeBytes(bytes: Buffer, charset: string | undefined) {+ if (charset) {+ try {+ return new TextDecoder(charset).decode(bytes);+ } catch {+ // unknown label: read as UTF-8+ }+ }+ return bytes.toString("utf8");+}+function decodeBody(+ body: string,+ encoding: string | undefined,+ charset: string | undefined,+) {const kind = encoding?.toLowerCase();if (kind === "base64")- return Buffer.from(body.replace(/\s+/g, ""), "base64").toString("utf8");+ return decodeBytes(+ Buffer.from(body.replace(/\s+/g, ""), "base64"),+ charset,+ );if (kind === "quoted-printable")- return Buffer.from(- body- .replace(/=\r?\n/g, "")- .replace(/=([0-9A-F]{2})/gi, (_, hex: string) =>- String.fromCharCode(Number.parseInt(hex, 16)),- ),- "latin1",- ).toString("utf8");+ return decodeBytes(+ Buffer.from(+ body+ .replace(/=\r?\n/g, "")+ .replace(/=([0-9A-F]{2})/gi, (_, hex: string) =>+ String.fromCharCode(Number.parseInt(hex, 16)),+ ),+ "latin1",+ ),+ charset,+ );return body;}/**@@ -229,7 +256,11 @@ export function extractEmailText(raw: string, depth = 0): string {plain ?? texts.find((part) => /^text\/html/i.test(part.type));return chosen ? extractEmailText(chosen.part, depth + 1) : "";}- const decoded = decodeBody(body, headers.get("content-transfer-encoding"));+ const decoded = decodeBody(+ body,+ headers.get("content-transfer-encoding"),+ /charset="?([^";\s]+)"?/i.exec(type)?.[1],+ );return /^text\/html/i.test(type)? decoded.replace(/<style[\s\S]*?<\/style>|<script[\s\S]*?<\/script>/gi, "")diff --git a/server/tests/responsibility-triggers.test.ts b/server/tests/responsibility-triggers.test.tsindex a36232a5..d115422f 100644--- a/server/tests/responsibility-triggers.test.ts+++ b/server/tests/responsibility-triggers.test.ts@@ -706,6 +706,29 @@ describe("inbound email via SES -> SNS", () => {'Q3 numbers for AT&T: margin < 5%, we’re "on track" — < stays',);});+ test("MIME text extraction decodes in the charset the part names", () => {+ const latin1 = [+ "Content-Type: text/plain; charset=iso-8859-1",+ "Content-Transfer-Encoding: quoted-printable",+ "",+ "Caf=E9 ferm=E9 =E0 minuit.",+ ].join("\r\n");+ expect(extractEmailText(latin1)).toBe("Café fermé à minuit.");+ const base64 = [+ 'Content-Type: text/plain; charset="windows-1252"',+ "Content-Transfer-Encoding: base64",+ "",+ Buffer.from("Na\xefve r\xe9sum\xe9", "latin1").toString("base64"),+ ].join("\r\n");+ expect(extractEmailText(base64)).toBe("Naïve résumé");+ const utf8 = [+ "Content-Type: text/plain; charset=utf-8",+ "Content-Transfer-Encoding: quoted-printable",+ "",+ "Caf=C3=A9",+ ].join("\r\n");+ expect(extractEmailText(utf8)).toBe("Café");+ });test("signing certificate URLs must be HTTPS on an sns.<region>.amazonaws.com host", () => {expect(isSnsUrl(CERT_URL, true)).toBe(true);expect(