Skip to content

armgit / posta

Public Imported from GitHub
0

Self-hosted email delivery platform that enables applications to send emails via HTTP APIs while handling SMTP delivery, templates, storage, security, and analytics. Fully self-hostable alternative to services like SendGrid or Mailgun.

fix(attachments): keep payload for queued sends without blob storage

Mike Kosulin4cd050038ab0
diff --git a/internal/services/email/attachment_pipeline_test.go b/internal/services/email/attachment_pipeline_test.go
new file mode 100644
index 0000000..102b931
--- /dev/null
+++ b/internal/services/email/attachment_pipeline_test.go
@@ -0,0 +1,249 @@
+/*
+ * Copyright 2026 Jonas Kaninda
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ */
+
+package email
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "io"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/goposta/posta/internal/models"
+ "github.com/goposta/posta/internal/services/ratelimit"
+ "github.com/goposta/posta/internal/storage/repositories"
+ "github.com/redis/go-redis/v9"
+ "gorm.io/driver/postgres"
+ "gorm.io/gorm"
+ gormlogger "gorm.io/gorm/logger"
+)
+
+// stubEnqueuer stands in for the Asynq producer: it records the enqueued email id
+// and, crucially, sends nothing itself — exactly like production, where delivery
+// happens later in a worker that has only the stored record to work from.
+type stubEnqueuer struct{ enqueued []uint }
+
+func (s *stubEnqueuer) EnqueueEmailSend(emailID uint, _ string) error {
+ s.enqueued = append(s.enqueued, emailID)
+ return nil
+}
+
+func (s *stubEnqueuer) EnqueueEmailSendAt(emailID uint, _ string, _ time.Time) error {
+ s.enqueued = append(s.enqueued, emailID)
+ return nil
+}
+
+func pipelineTestDB(t *testing.T) *gorm.DB {
+ t.Helper()
+ dsn := os.Getenv("TEST_DATABASE_DSN")
+ if dsn == "" {
+ dsn = "host=localhost user=posta password=posta dbname=posta port=5432 sslmode=disable"
+ }
+ db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: gormlogger.Default.LogMode(gormlogger.Silent)})
+ if err != nil {
+ t.Skipf("skipping: no test database available: %v", err)
+ }
+ if err := db.Exec(`CREATE EXTENSION IF NOT EXISTS pgcrypto`).Error; err != nil {
+ t.Skipf("skipping: cannot enable pgcrypto: %v", err)
+ }
+ if err := db.AutoMigrate(&models.User{}, &models.Email{}); err != nil {
+ t.Skipf("skipping: cannot migrate test schema: %v", err)
+ }
+ return db
+}
+
+func pipelineTestLimiter(t *testing.T) *ratelimit.RedisLimiter {
+ t.Helper()
+ addr := os.Getenv("TEST_REDIS_ADDR")
+ if addr == "" {
+ addr = "localhost:6379"
+ }
+ client := redis.NewClient(&redis.Options{Addr: addr})
+ if err := client.Ping(context.Background()).Err(); err != nil {
+ t.Skipf("skipping: no test redis available: %v", err)
+ }
+ return ratelimit.NewRedisLimiter(client, 100000, 100000)
+}
+
+// Reproduces the production failure behind DashaMail's "Невозможно прочитать файл!":
+// an email submitted with an attachment is queued for the worker, and the worker
+// only ever sees the stored record. If the payload is not recoverable from that
+// record the attachment goes out as a zero-byte file.
+//
+// This is the SMTP relay's only path — the relay always goes through Send, which
+// always enqueues.
+func TestQueuedEmailKeepsRecoverableAttachment(t *testing.T) {
+ db := pipelineTestDB(t)
+ tx := db.Begin()
+ t.Cleanup(func() { tx.Rollback() })
+
+ user := &models.User{Name: "relay", Email: "[email protected]", PasswordHash: "x"}
+ if err := tx.Create(user).Error; err != nil {
+ t.Fatalf("create user: %v", err)
+ }
+
+ enq := &stubEnqueuer{}
+ svc := &Service{
+ emailRepo: repositories.NewEmailRepository(tx),
+ limiter: pipelineTestLimiter(t),
+ enqueuer: enq,
+ // blobStore intentionally nil: POSTA_BLOB_PROVIDER is empty by default.
+ }
+
+ payload := []byte("delivery check 2026-07-27 12:05:51 +0300\n")
+ resp, err := svc.Send(context.Background(), user.ID, 0, nil, user.Email, &SendRequest{
+ To: []string{"[email protected]"},
+ Subject: "Проверка вложения",
+ Text: "Файл во вложении",
+ Attachments: []models.Attachment{{
+ Filename: "delivery-check-1785143151.txt",
+ ContentType: "text/plain",
+ Content: base64.StdEncoding.EncodeToString(payload),
+ }},
+ })
+ if err != nil {
+ t.Fatalf("Send: %v", err)
+ }
+ if resp.Status != models.EmailStatusQueued {
+ t.Fatalf("status = %q, want queued", resp.Status)
+ }
+ if len(enq.enqueued) != 1 {
+ t.Fatalf("enqueued %d emails, want 1", len(enq.enqueued))
+ }
+
+ // Reload exactly what the worker will read.
+ em, err := repositories.NewEmailRepository(tx).FindByID(enq.enqueued[0])
+ if err != nil {
+ t.Fatalf("reload email: %v", err)
+ }
+
+ atts, err := LoadAttachments(context.Background(), em, nil)
+ if err != nil {
+ t.Fatalf("LoadAttachments: %v", err)
+ }
+ if len(atts) != 1 {
+ t.Fatalf("recovered %d attachments, want 1", len(atts))
+ }
+ if atts[0].Content == "" {
+ t.Fatalf("attachment payload was lost: the worker would send a zero-byte file "+
+ "(record attachments_json=%q)", em.AttachmentsJSON)
+ }
+ got, err := base64.StdEncoding.DecodeString(atts[0].Content)
+ if err != nil {
+ t.Fatalf("recovered content is not valid base64: %v", err)
+ }
+ if string(got) != string(payload) {
+ t.Errorf("recovered payload = %q, want %q", got, payload)
+ }
+ if atts[0].Filename != "delivery-check-1785143151.txt" {
+ t.Errorf("filename = %q, want delivery-check-1785143151.txt", atts[0].Filename)
+ }
+ if atts[0].ContentType != "text/plain" {
+ t.Errorf("content type = %q, want text/plain", atts[0].ContentType)
+ }
+}
+
+// With a blob store the bytes go to the store and the record keeps only the key,
+// so the payload never sits in the database.
+func TestQueuedEmailStoresAttachmentInBlobStore(t *testing.T) {
+ db := pipelineTestDB(t)
+ tx := db.Begin()
+ t.Cleanup(func() { tx.Rollback() })
+
+ user := &models.User{Name: "relay", Email: "[email protected]", PasswordHash: "x"}
+ if err := tx.Create(user).Error; err != nil {
+ t.Fatalf("create user: %v", err)
+ }
+
+ store := &memBlobStore{objects: map[string][]byte{}}
+ enq := &stubEnqueuer{}
+ svc := &Service{
+ emailRepo: repositories.NewEmailRepository(tx),
+ limiter: pipelineTestLimiter(t),
+ enqueuer: enq,
+ blobStore: store,
+ }
+
+ payload := []byte("delivery check")
+ if _, err := svc.Send(context.Background(), user.ID, 0, nil, user.Email, &SendRequest{
+ To: []string{"[email protected]"},
+ Subject: "Проверка вложения",
+ Text: "Файл во вложении",
+ Attachments: []models.Attachment{{
+ Filename: "delivery-check.txt",
+ ContentType: "text/plain",
+ Content: base64.StdEncoding.EncodeToString(payload),
+ }},
+ }); err != nil {
+ t.Fatalf("Send: %v", err)
+ }
+
+ em, err := repositories.NewEmailRepository(tx).FindByID(enq.enqueued[0])
+ if err != nil {
+ t.Fatalf("reload email: %v", err)
+ }
+
+ var meta []map[string]any
+ if err := json.Unmarshal([]byte(em.AttachmentsJSON), &meta); err != nil {
+ t.Fatalf("attachments_json is not valid JSON: %v", err)
+ }
+ if content, _ := meta[0]["content"].(string); content != "" {
+ t.Errorf("payload should live in the blob store, not the record: %s", em.AttachmentsJSON)
+ }
+
+ atts, err := LoadAttachments(context.Background(), em, store)
+ if err != nil {
+ t.Fatalf("LoadAttachments: %v", err)
+ }
+ got, err := base64.StdEncoding.DecodeString(atts[0].Content)
+ if err != nil {
+ t.Fatalf("recovered content is not valid base64: %v", err)
+ }
+ if string(got) != string(payload) {
+ t.Errorf("recovered payload = %q, want %q", got, payload)
+ }
+}
+
+// memBlobStore is an in-memory blob.Store for the blob-backed path.
+type memBlobStore struct{ objects map[string][]byte }
+
+func (m *memBlobStore) Put(_ context.Context, key string, r io.Reader, _ string) error {
+ data, err := io.ReadAll(r)
+ if err != nil {
+ return err
+ }
+ m.objects[key] = data
+ return nil
+}
+
+func (m *memBlobStore) Get(_ context.Context, key string) (io.ReadCloser, error) {
+ data, ok := m.objects[key]
+ if !ok {
+ return nil, errors.New("not found")
+ }
+ return io.NopCloser(bytes.NewReader(data)), nil
+}
+
+func (m *memBlobStore) Delete(_ context.Context, key string) error {
+ delete(m.objects, key)
+ return nil
+}
+
+func (m *memBlobStore) Exists(_ context.Context, key string) (bool, error) {
+ _, ok := m.objects[key]
+ return ok, nil
+}
diff --git a/internal/services/email/service.go b/internal/services/email/service.go
index 5afe255..104e5ed 100644
--- a/internal/services/email/service.go
+++ b/internal/services/email/service.go
@@ -206,13 +206,17 @@ func (s *Service) uploadAttachments(ctx context.Context, emailUUID string, attac
// DownloadAttachments fetches attachment content from blob storage and populates
// the base64 Content field for each attachment that has a StorageKey.
func (s *Service) DownloadAttachments(ctx context.Context, attachments []models.Attachment) ([]models.Attachment, error) {
+ return fetchAttachmentContent(ctx, s.blobStore, attachments)
+}
+
+func fetchAttachmentContent(ctx context.Context, store blob.Store, attachments []models.Attachment) ([]models.Attachment, error) {
result := make([]models.Attachment, len(attachments))
for i, att := range attachments {
result[i] = att
if att.StorageKey == "" {
continue
}
- rc, err := s.blobStore.Get(ctx, att.StorageKey)
+ rc, err := store.Get(ctx, att.StorageKey)
if err != nil {
return nil, fmt.Errorf("failed to download attachment %q: %w", att.Filename, err)
}
@@ -226,6 +230,42 @@ func (s *Service) DownloadAttachments(ctx context.Context, attachments []models.
return result, nil
}
+// persistAttachments prepares attachments for the email record. With a blob store
+// the bytes are uploaded and the record keeps only the key; without one the base64
+// content stays on the record, because the record is all the send worker gets.
+func (s *Service) persistAttachments(ctx context.Context, keyPrefix string, atts []models.Attachment) (string, error) {
+ if len(atts) == 0 {
+ return "", nil
+ }
+ stored := atts
+ if s.blobStore != nil {
+ uploaded, err := s.uploadAttachments(ctx, keyPrefix, atts)
+ if err != nil {
+ return "", fmt.Errorf("attachment upload: %w", err)
+ }
+ stored = uploaded
+ }
+ b, _ := json.Marshal(stored)
+ return string(b), nil
+}
+
+// LoadAttachments rebuilds the attachments to send from a stored email record.
+// The send worker runs long after Send returned, so the record is all it has:
+// whatever cannot be recovered here goes out as a zero-byte file. store may be nil.
+func LoadAttachments(ctx context.Context, em *models.Email, store blob.Store) ([]models.Attachment, error) {
+ if em.AttachmentsJSON == "" {
+ return nil, nil
+ }
+ var atts []models.Attachment
+ if err := json.Unmarshal([]byte(em.AttachmentsJSON), &atts); err != nil {
+ return nil, fmt.Errorf("decode attachments: %w", err)
+ }
+ if store == nil {
+ return atts, nil
+ }
+ return fetchAttachmentContent(ctx, store, atts)
+}
+
// DeleteAttachments removes attachment blobs from storage for the given keys.
func (s *Service) DeleteAttachments(ctx context.Context, attachments []models.Attachment) {
for _, att := range attachments {
@@ -676,32 +716,11 @@ func (s *Service) Send(ctx context.Context, userID, apiKeyID uint, workspaceID *
req.To = filtered
}
- // Serialize attachments for storage. When blob storage is configured,
- // upload the binary content and store only the storage key reference.
- // Otherwise store just filename + content_type metadata.
- var attachmentsJSON string
- if len(req.Attachments) > 0 {
- if s.blobStore != nil {
- // Generate a temporary UUID for the blob key prefix.
- tempKey := fmt.Sprintf("%d", time.Now().UnixNano())
- uploaded, err := s.uploadAttachments(ctx, tempKey, req.Attachments)
- if err != nil {
- return nil, fmt.Errorf("attachment upload: %w", err)
- }
- b, _ := json.Marshal(uploaded)
- attachmentsJSON = string(b)
- } else {
- type attachMeta struct {
- Filename string `json:"filename"`
- ContentType string `json:"content_type"`
- }
- meta := make([]attachMeta, len(req.Attachments))
- for i, a := range req.Attachments {
- meta[i] = attachMeta{Filename: a.Filename, ContentType: a.ContentType}
- }
- b, _ := json.Marshal(meta)
- attachmentsJSON = string(b)
- }
+ // Persist attachments. The blob key prefix is a temporary stand-in for the
+ // email UUID, which is not assigned until the record is created.
+ attachmentsJSON, err := s.persistAttachments(ctx, fmt.Sprintf("%d", time.Now().UnixNano()), req.Attachments)
+ if err != nil {
+ return nil, err
}
// Auto-generate plain text from HTML when text body is not provided
diff --git a/internal/worker/handler.go b/internal/worker/handler.go
index 260ad41..25eb6cd 100644
--- a/internal/worker/handler.go
+++ b/internal/worker/handler.go
@@ -19,11 +19,9 @@ package worker
import (
"context"
- "encoding/base64"
"encoding/json"
"errors"
"fmt"
- "io"
"net/mail"
"strings"
"time"
@@ -233,27 +231,17 @@ func (h *EmailSendHandler) ProcessTask(ctx context.Context, t *asynq.Task) error
em.TextBody = email.HTMLToText(em.HTMLBody)
}
- // Parse attachments from stored JSON and fetch content from blob storage if needed
- var attachments []models.Attachment
- if em.AttachmentsJSON != "" {
- _ = json.Unmarshal([]byte(em.AttachmentsJSON), &attachments)
- if h.blobStore != nil {
- for i, att := range attachments {
- if att.StorageKey != "" && att.Content == "" {
- rc, err := h.blobStore.Get(ctx, att.StorageKey)
- if err != nil {
- h.markFailed(em, fmt.Sprintf("failed to fetch attachment %q from storage: %v", att.Filename, err))
- return nil
- }
- data, err := io.ReadAll(rc)
- _ = rc.Close()
- if err != nil {
- h.markFailed(em, fmt.Sprintf("failed to read attachment %q: %v", att.Filename, err))
- return nil
- }
- attachments[i].Content = base64.StdEncoding.EncodeToString(data)
- }
- }
+ // Rebuild the attachments from the record, pulling blob-stored content if needed.
+ attachments, err := email.LoadAttachments(ctx, em, h.blobStore)
+ if err != nil {
+ h.markFailed(em, fmt.Sprintf("failed to load attachments: %v", err))
+ return nil
+ }
+ // An attachment with no payload would go out as a zero-byte file; fail loudly instead.
+ for _, att := range attachments {
+ if att.Content == "" {
+ h.markFailed(em, fmt.Sprintf("attachment %q has no content", att.Filename))
+ return nil
}
}