Skip to content

armgit / posta

Public Imported from GitHub
0

Self-hosted email delivery platform that enables applications to send emails via HTTP APIs while handling SMTP delivery, templates, storage, security, and analytics. Fully self-hostable alternative to services like SendGrid or Mailgun.

fix(inbound): decode non-UTF-8 email bodies and headers before persisting Fixes #71

Jonas Kanindab46c69d8e71d
diff --git a/internal/services/inbound/parser.go b/internal/services/inbound/parser.go
index 14e3c8c..38f3af6 100644
--- a/internal/services/inbound/parser.go
+++ b/internal/services/inbound/parser.go
@@ -28,6 +28,9 @@ import (
"net/mail"
"strings"
"time"
+ "unicode/utf8"
+
+ "golang.org/x/net/html/charset"
)
const contentTypeTextPlain = "text/plain"
@@ -69,7 +72,7 @@ func ParseRawEmail(raw []byte) (*ParsedEmail, error) {
Raw: raw,
}
- dec := &mime.WordDecoder{}
+ dec := newWordDecoder()
for k, vs := range msg.Header {
if len(vs) == 0 {
continue
@@ -78,7 +81,7 @@ func ParseRawEmail(raw []byte) (*ParsedEmail, error) {
if decoded, derr := dec.DecodeHeader(v); derr == nil {
v = decoded
}
- p.Headers[k] = v
+ p.Headers[k] = ensureUTF8(v)
}
p.MessageID = strings.Trim(p.Headers["Message-Id"], "<>")
@@ -209,7 +212,7 @@ func assignBody(p *ParsedEmail, mediaType, body string) {
}
}
-func readBody(r io.Reader, encoding, _ string) (string, error) {
+func readBody(r io.Reader, encoding, charsetName string) (string, error) {
raw, err := io.ReadAll(r)
if err != nil {
return "", fmt.Errorf("read body: %w", err)
@@ -218,7 +221,42 @@ func readBody(r io.Reader, encoding, _ string) (string, error) {
if err != nil {
return "", err
}
- return string(decoded), nil
+ return decodeCharset(decoded, charsetName), nil
+}
+
+func decodeCharset(b []byte, charsetName string) string {
+ if charsetName != "" {
+ if enc, _ := charset.Lookup(charsetName); enc != nil {
+ if out, err := enc.NewDecoder().Bytes(b); err == nil && utf8.Valid(out) {
+ return string(out)
+ }
+ }
+ }
+ return ensureUTF8(string(b))
+}
+
+func ensureUTF8(s string) string {
+ if utf8.ValidString(s) {
+ return s
+ }
+ var sb strings.Builder
+ sb.Grow(len(s))
+ for i := 0; i < len(s); i++ {
+ sb.WriteRune(rune(s[i]))
+ }
+ return sb.String()
+}
+
+func newWordDecoder() *mime.WordDecoder {
+ return &mime.WordDecoder{
+ CharsetReader: func(label string, input io.Reader) (io.Reader, error) {
+ enc, _ := charset.Lookup(label)
+ if enc == nil {
+ return input, nil
+ }
+ return enc.NewDecoder().Reader(input), nil
+ },
+ }
}
func decodeTransfer(raw []byte, encoding string) ([]byte, error) {
@@ -244,11 +282,11 @@ func decodeHeaderValue(v string) string {
if v == "" {
return ""
}
- dec := &mime.WordDecoder{}
+ dec := newWordDecoder()
if out, err := dec.DecodeHeader(v); err == nil {
- return out
+ return ensureUTF8(out)
}
- return v
+ return ensureUTF8(v)
}
func headerAddress(v string) string {
diff --git a/internal/services/inbound/parser_test.go b/internal/services/inbound/parser_test.go
index e531042..f8a6b86 100644
--- a/internal/services/inbound/parser_test.go
+++ b/internal/services/inbound/parser_test.go
@@ -13,6 +13,7 @@ package inbound
import (
"strings"
"testing"
+ "unicode/utf8"
)
func TestParsePlainText(t *testing.T) {
@@ -100,6 +101,62 @@ func TestParseMultipartWithAttachment(t *testing.T) {
}
}
+func TestParseISO88591Body(t *testing.T) {
+ body := []byte{'G', 'r', 0xfc, 0xdf, 'e', '\r', '\n'}
+ raw := append([]byte(
+ "From: [email protected]\r\n"+
+ "To: [email protected]\r\n"+
+ "Subject: Hi\r\n"+
+ "Content-Type: text/plain; charset=iso-8859-1\r\n"+
+ "\r\n"), body...)
+
+ p, err := ParseRawEmail(raw)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+ if !utf8.ValidString(p.TextBody) {
+ t.Errorf("TextBody is not valid UTF-8: %q", p.TextBody)
+ }
+ if !strings.Contains(p.TextBody, "Grüße") {
+ t.Errorf("TextBody = %q, want to contain Grüße", p.TextBody)
+ }
+}
+
+func TestParseISO88591EncodedHeader(t *testing.T) {
+ raw := "From: [email protected]\r\n" +
+ "To: [email protected]\r\n" +
+ "Subject: =?iso-8859-1?Q?Gr=FC=DFe?=\r\n" +
+ "Content-Type: text/plain; charset=UTF-8\r\n" +
+ "\r\n" +
+ "hi\r\n"
+
+ p, err := ParseRawEmail([]byte(raw))
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+ if p.Subject != "Grüße" {
+ t.Errorf("Subject = %q, want Grüße", p.Subject)
+ }
+}
+
+func TestParseUndeclaredNonUTF8BodyFallsBackToLatin1(t *testing.T) {
+ body := []byte{0xdf, 0x65, '\r', '\n'}
+ raw := append([]byte(
+ "From: [email protected]\r\n"+
+ "To: [email protected]\r\n"+
+ "Subject: Hi\r\n"+
+ "Content-Type: text/plain\r\n"+
+ "\r\n"), body...)
+
+ p, err := ParseRawEmail(raw)
+ if err != nil {
+ t.Fatalf("parse: %v", err)
+ }
+ if !utf8.ValidString(p.TextBody) {
+ t.Errorf("TextBody is not valid UTF-8: %q", p.TextBody)
+ }
+}
+
func TestParseQuotedPrintable(t *testing.T) {
raw := "From: [email protected]\r\n" +
"To: [email protected]\r\n" +