Self-hosted email delivery platform that enables applications to send emails via HTTP APIs while handling SMTP delivery, templates, storage, security, and analytics. Fully self-hostable alternative to services like SendGrid or Mailgun.
Go email-relayemail-routinghttp-to-smtpmail-deliverymailchimp-alternativemailgun-alternativemailsendermailservermailtemplatemailtestpostmark-alternativesend-emailsendgrid-alternativesendmailtransactional-emailtransactional-outbox
fix(inbound): decode non-UTF-8 email bodies and headers before persisting Fixes #71
Jonas Kanindab46c69d8e71d
diff --git a/internal/services/inbound/parser.go b/internal/services/inbound/parser.goindex 14e3c8c..38f3af6 100644--- a/internal/services/inbound/parser.go+++ b/internal/services/inbound/parser.go@@ -28,6 +28,9 @@ import ("net/mail""strings""time"+ "unicode/utf8"++ "golang.org/x/net/html/charset")const contentTypeTextPlain = "text/plain"@@ -69,7 +72,7 @@ func ParseRawEmail(raw []byte) (*ParsedEmail, error) {Raw: raw,}- dec := &mime.WordDecoder{}+ dec := newWordDecoder()for k, vs := range msg.Header {if len(vs) == 0 {continue@@ -78,7 +81,7 @@ func ParseRawEmail(raw []byte) (*ParsedEmail, error) {if decoded, derr := dec.DecodeHeader(v); derr == nil {v = decoded}- p.Headers[k] = v+ p.Headers[k] = ensureUTF8(v)}p.MessageID = strings.Trim(p.Headers["Message-Id"], "<>")@@ -209,7 +212,7 @@ func assignBody(p *ParsedEmail, mediaType, body string) {}}-func readBody(r io.Reader, encoding, _ string) (string, error) {+func readBody(r io.Reader, encoding, charsetName string) (string, error) {raw, err := io.ReadAll(r)if err != nil {return "", fmt.Errorf("read body: %w", err)@@ -218,7 +221,42 @@ func readBody(r io.Reader, encoding, _ string) (string, error) {if err != nil {return "", err}- return string(decoded), nil+ return decodeCharset(decoded, charsetName), nil+}++func decodeCharset(b []byte, charsetName string) string {+ if charsetName != "" {+ if enc, _ := charset.Lookup(charsetName); enc != nil {+ if out, err := enc.NewDecoder().Bytes(b); err == nil && utf8.Valid(out) {+ return string(out)+ }+ }+ }+ return ensureUTF8(string(b))+}++func ensureUTF8(s string) string {+ if utf8.ValidString(s) {+ return s+ }+ var sb strings.Builder+ sb.Grow(len(s))+ for i := 0; i < len(s); i++ {+ sb.WriteRune(rune(s[i]))+ }+ return sb.String()+}++func newWordDecoder() *mime.WordDecoder {+ return &mime.WordDecoder{+ CharsetReader: func(label string, input io.Reader) (io.Reader, error) {+ enc, _ := charset.Lookup(label)+ if enc == nil {+ return input, nil+ }+ return enc.NewDecoder().Reader(input), nil+ },+ }}func decodeTransfer(raw []byte, encoding string) ([]byte, error) {@@ -244,11 +282,11 @@ func decodeHeaderValue(v string) string {if v == "" {return ""}- dec := &mime.WordDecoder{}+ dec := newWordDecoder()if out, err := dec.DecodeHeader(v); err == nil {- return out+ return ensureUTF8(out)}- return v+ return ensureUTF8(v)}func headerAddress(v string) string {diff --git a/internal/services/inbound/parser_test.go b/internal/services/inbound/parser_test.goindex e531042..f8a6b86 100644--- a/internal/services/inbound/parser_test.go+++ b/internal/services/inbound/parser_test.go@@ -13,6 +13,7 @@ package inboundimport ("strings""testing"+ "unicode/utf8")func TestParsePlainText(t *testing.T) {@@ -100,6 +101,62 @@ func TestParseMultipartWithAttachment(t *testing.T) {}}+func TestParseISO88591Body(t *testing.T) {+ body := []byte{'G', 'r', 0xfc, 0xdf, 'e', '\r', '\n'}+ raw := append([]byte(+ "From: [email protected]\r\n"++ "To: [email protected]\r\n"++ "Subject: Hi\r\n"++ "Content-Type: text/plain; charset=iso-8859-1\r\n"++ "\r\n"), body...)++ p, err := ParseRawEmail(raw)+ if err != nil {+ t.Fatalf("parse: %v", err)+ }+ if !utf8.ValidString(p.TextBody) {+ t.Errorf("TextBody is not valid UTF-8: %q", p.TextBody)+ }+ if !strings.Contains(p.TextBody, "Grüße") {+ t.Errorf("TextBody = %q, want to contain Grüße", p.TextBody)+ }+}++func TestParseISO88591EncodedHeader(t *testing.T) {+ raw := "From: [email protected]\r\n" ++ "To: [email protected]\r\n" ++ "Subject: =?iso-8859-1?Q?Gr=FC=DFe?=\r\n" ++ "Content-Type: text/plain; charset=UTF-8\r\n" ++ "\r\n" ++ "hi\r\n"++ p, err := ParseRawEmail([]byte(raw))+ if err != nil {+ t.Fatalf("parse: %v", err)+ }+ if p.Subject != "Grüße" {+ t.Errorf("Subject = %q, want Grüße", p.Subject)+ }+}++func TestParseUndeclaredNonUTF8BodyFallsBackToLatin1(t *testing.T) {+ body := []byte{0xdf, 0x65, '\r', '\n'}+ raw := append([]byte(+ "From: [email protected]\r\n"++ "To: [email protected]\r\n"++ "Subject: Hi\r\n"++ "Content-Type: text/plain\r\n"++ "\r\n"), body...)++ p, err := ParseRawEmail(raw)+ if err != nil {+ t.Fatalf("parse: %v", err)+ }+ if !utf8.ValidString(p.TextBody) {+ t.Errorf("TextBody is not valid UTF-8: %q", p.TextBody)+ }+}+func TestParseQuotedPrintable(t *testing.T) {raw := "From: [email protected]\r\n" +"To: [email protected]\r\n" +