Skip to content

armgit / posta

Public Imported from GitHub
0

Self-hosted email delivery platform that enables applications to send emails via HTTP APIs while handling SMTP delivery, templates, storage, security, and analytics. Fully self-hostable alternative to services like SendGrid or Mailgun.

feat(inbound): store raw message before processing to prevent email loss. (#79)

Jonas Kaninda9be725950699
diff --git a/cmd/posta/server.go b/cmd/posta/server.go
index 0a15f7d..8e5166f 100644
--- a/cmd/posta/server.go
+++ b/cmd/posta/server.go
@@ -118,16 +118,22 @@ func runServer(cli *okapicli.CLI) {
logger.Info("system notification service enabled")
}
+ // Bus is shared between the embedded worker (parse handler) and the
+ // SMTP server so a single PublishSimple reaches both sets of
+ // in-process subscribers.
+ var inboundBus *eventbus.EventBus
+ if cfg.InboundEnabled && !cfg.DevMode {
+ inboundBus = eventbus.New(repositories.NewEventRepository(res.db))
+ }
+
if !cfg.DevMode {
res.producer = worker.NewProducer(cfg.Redis.Addr, cfg.Redis.Password, cfg.WorkerMaxRetries)
if cfg.EmbeddedWorker {
- startEmbeddedWorker(res.db, cfg, res.blobStore, notifier)
+ startEmbeddedWorker(res.db, cfg, res.blobStore, notifier, inboundBus)
}
}
if cfg.InboundEnabled && !cfg.DevMode {
- eventRepo := repositories.NewEventRepository(res.db)
- inboundBus := eventbus.New(eventRepo)
if srv, err := startInboundSMTPServer(res.db, cfg, res.blobStore, res.producer, inboundBus); err != nil {
logger.Error("failed to start inbound SMTP server", "error", err)
} else {
@@ -200,6 +206,32 @@ func newWebhookDispatcher(db *gorm.DB, cfg *config.Config) *webhook.Dispatcher {
return dispatcher
}
+func newInboundServiceForWorker(db *gorm.DB,
+ cfg *config.Config,
+ blobStore blob.Store,
+ producer *worker.Producer,
+ bus *eventbus.EventBus) *inbound.Service {
+ svc := inbound.NewService(
+ repositories.NewInboundEmailRepository(db),
+ repositories.NewDomainRepository(db),
+ repositories.NewSuppressionRepository(db),
+ inbound.Config{
+ MaxMessageSize: cfg.InboundMaxMessageSize,
+ MaxAttachmentSize: cfg.InboundMaxAttachSize,
+ },
+ )
+ if blobStore != nil {
+ svc.SetBlobStore(blobStore)
+ }
+ if producer != nil {
+ svc.SetEnqueuer(producer)
+ }
+ if bus != nil {
+ svc.SetEventBus(bus)
+ }
+ return svc
+}
+
// startInboundSMTPServer configures and launches the built-in SMTP receiver.
// Returns the server so it can be gracefully shut down on exit.
func startInboundSMTPServer(
@@ -263,7 +295,13 @@ func startInboundSMTPServer(
}
// startEmbeddedWorker starts an in-process asynq worker for email delivery.
-func startEmbeddedWorker(db *gorm.DB, cfg *config.Config, blobStore blob.Store, notifier *notification.Service) {
+// The bus argument is used by the inbound parse handler to publish
+// email.inbound.received events; it may be nil when inbound is disabled.
+func startEmbeddedWorker(db *gorm.DB,
+ cfg *config.Config,
+ blobStore blob.Store,
+ notifier *notification.Service,
+ bus *eventbus.EventBus) {
dispatcher := newWebhookDispatcher(db, cfg)
handler := worker.NewEmailSendHandler(
@@ -359,6 +397,15 @@ func startEmbeddedWorker(db *gorm.DB, cfg *config.Config, blobStore blob.Store,
inboundHandler.OnForwarded(metrics.IncrementInboundForwarded)
inboundHandler.OnFailed(metrics.IncrementInboundFailed)
mux.HandleFunc(worker.TypeInboundProcess, inboundHandler.ProcessTask)
+
+ parseProducer := worker.NewProducer(cfg.Redis.Addr, cfg.Redis.Password, cfg.WorkerMaxRetries)
+ parseSvc := newInboundServiceForWorker(db, cfg, blobStore, parseProducer, bus)
+ parseHandler := worker.NewInboundParseHandler(
+ repositories.NewInboundEmailRepository(db),
+ parseSvc,
+ parseProducer,
+ )
+ mux.HandleFunc(worker.TypeInboundParse, parseHandler.ProcessTask)
}
go func() {
diff --git a/cmd/posta/worker.go b/cmd/posta/worker.go
index ff9a1f7..7aa8074 100644
--- a/cmd/posta/worker.go
+++ b/cmd/posta/worker.go
@@ -23,6 +23,8 @@ import (
"github.com/goposta/posta/internal/metrics"
"github.com/goposta/posta/internal/services/crypto"
"github.com/goposta/posta/internal/services/email"
+ "github.com/goposta/posta/internal/services/eventbus"
+ "github.com/goposta/posta/internal/services/inbound"
"github.com/goposta/posta/internal/services/notification"
"github.com/goposta/posta/internal/services/tracking"
"github.com/goposta/posta/internal/storage/blob"
@@ -168,6 +170,29 @@ func runWorker() error {
inboundHandler.OnForwarded(metrics.IncrementInboundForwarded)
inboundHandler.OnFailed(metrics.IncrementInboundFailed)
mux.HandleFunc(worker.TypeInboundProcess, inboundHandler.ProcessTask)
+
+ parseProducer := worker.NewProducer(cfg.Redis.Addr, cfg.Redis.Password, cfg.WorkerMaxRetries)
+ parseBus := eventbus.New(repositories.NewEventRepository(db))
+ parseSvc := inbound.NewService(
+ repositories.NewInboundEmailRepository(db),
+ repositories.NewDomainRepository(db),
+ repositories.NewSuppressionRepository(db),
+ inbound.Config{
+ MaxMessageSize: cfg.InboundMaxMessageSize,
+ MaxAttachmentSize: cfg.InboundMaxAttachSize,
+ },
+ )
+ if blobStore != nil {
+ parseSvc.SetBlobStore(blobStore)
+ }
+ parseSvc.SetEnqueuer(parseProducer)
+ parseSvc.SetEventBus(parseBus)
+ parseHandler := worker.NewInboundParseHandler(
+ repositories.NewInboundEmailRepository(db),
+ parseSvc,
+ parseProducer,
+ )
+ mux.HandleFunc(worker.TypeInboundParse, parseHandler.ProcessTask)
}
logger.Info("Posta worker started",
diff --git a/internal/handlers/inbound_handler.go b/internal/handlers/inbound_handler.go
index d0fa23b..a321f2b 100644
--- a/internal/handlers/inbound_handler.go
+++ b/internal/handlers/inbound_handler.go
@@ -337,7 +337,11 @@ func (h *InboundHandler) Delete(c *okapi.Context, req *GetEmailRequest) error {
return noContent(c)
}
-// Retry re-enqueues a failed inbound email for another webhook-dispatch attempt.
+// Retry re-enqueues an inbound email for another attempt. Quarantined records
+// (those that failed during parsing) are routed back through inbound:parse so
+// the operator can re-run the MIME pipeline after a fix is deployed; failed
+// or stuck records (parsed but webhook delivery exhausted) go back through
+// inbound:process for re-dispatch.
func (h *InboundHandler) Retry(c *okapi.Context, req *GetEmailRequest) error {
if h.producer == nil {
return c.AbortForbidden("inbound retry requires an async worker")
@@ -349,16 +353,27 @@ func (h *InboundHandler) Retry(c *okapi.Context, req *GetEmailRequest) error {
if !ownsResource(c, rec.UserID, rec.WorkspaceID) {
return c.AbortNotFound("inbound email not found")
}
- if rec.Status != models.InboundStatusFailed && rec.Status != models.InboundStatusReceived {
- return c.AbortBadRequest("only failed or stuck received messages can be retried")
- }
- rec.Status = models.InboundStatusReceived
- rec.ErrorMessage = ""
- if err := h.repo.Update(rec); err != nil {
- return c.AbortInternalServerError("failed to update inbound email", err)
- }
- if err := h.producer.EnqueueInboundProcess(rec.ID); err != nil {
- return c.AbortInternalServerError("failed to enqueue inbound task", err)
+ switch rec.Status {
+ case models.InboundStatusQuarantined:
+ rec.Status = models.InboundStatusReceived
+ rec.ErrorMessage = ""
+ if err := h.repo.Update(rec); err != nil {
+ return c.AbortInternalServerError("failed to update inbound email", err)
+ }
+ if err := h.producer.EnqueueInboundParse(rec.ID); err != nil {
+ return c.AbortInternalServerError("failed to enqueue inbound parse task", err)
+ }
+ case models.InboundStatusFailed, models.InboundStatusReceived:
+ rec.Status = models.InboundStatusReceived
+ rec.ErrorMessage = ""
+ if err := h.repo.Update(rec); err != nil {
+ return c.AbortInternalServerError("failed to update inbound email", err)
+ }
+ if err := h.producer.EnqueueInboundProcess(rec.ID); err != nil {
+ return c.AbortInternalServerError("failed to enqueue inbound task", err)
+ }
+ default:
+ return c.AbortBadRequest("only failed, quarantined, or stuck received messages can be retried")
}
return ok(c, map[string]string{"id": rec.UUID, "status": string(rec.Status)})
}
diff --git a/internal/models/inbound_email.go b/internal/models/inbound_email.go
index 79e050e..efd3530 100644
--- a/internal/models/inbound_email.go
+++ b/internal/models/inbound_email.go
@@ -26,10 +26,11 @@ import (
type InboundEmailStatus string
const (
- InboundStatusReceived InboundEmailStatus = "received"
- InboundStatusForwarded InboundEmailStatus = "forwarded"
- InboundStatusFailed InboundEmailStatus = "failed"
- InboundStatusRejected InboundEmailStatus = "rejected"
+ InboundStatusReceived InboundEmailStatus = "received"
+ InboundStatusForwarded InboundEmailStatus = "forwarded"
+ InboundStatusFailed InboundEmailStatus = "failed"
+ InboundStatusRejected InboundEmailStatus = "rejected"
+ InboundStatusQuarantined InboundEmailStatus = "quarantined"
)
type InboundSource string
@@ -40,30 +41,32 @@ const (
)
type InboundEmail struct {
- ID uint `json:"id" gorm:"primaryKey"`
- UUID string `json:"uuid" gorm:"type:uuid;default:gen_random_uuid();uniqueIndex;not null"`
- UserID uint `json:"user_id" gorm:"index;not null"`
- WorkspaceID *uint `json:"workspace_id,omitempty" gorm:"index"`
- DomainID uint `json:"domain_id" gorm:"index;not null"`
- MessageID string `json:"message_id" gorm:"index"`
- Sender string `json:"sender" gorm:"not null"`
- Recipients pq.StringArray `json:"recipients" gorm:"type:text[];not null"`
- Subject string `json:"subject"`
- TextBody string `json:"text_body" gorm:"type:text"`
- HTMLBody string `json:"html_body" gorm:"type:text"`
- AttachmentsJSON string `json:"attachments_json,omitempty" gorm:"type:text"`
- HeadersJSON string `json:"headers_json,omitempty" gorm:"type:text"`
- RawStorageKey string `json:"raw_storage_key,omitempty"`
- DedupHash string `json:"-" gorm:"index"`
- Size int64 `json:"size"`
- SpamScore *float64 `json:"spam_score,omitempty"`
- Status InboundEmailStatus `json:"status" gorm:"default:received;not null"`
- Source InboundSource `json:"source" gorm:"not null"`
- ErrorMessage string `json:"error_message,omitempty"`
- RetryCount int `json:"retry_count" gorm:"default:0;not null"`
- ReceivedAt time.Time `json:"received_at" gorm:"not null"`
- ForwardedAt *time.Time `json:"forwarded_at,omitempty"`
- CreatedAt time.Time `json:"created_at"`
+ ID uint `json:"id" gorm:"primaryKey"`
+ UUID string `json:"uuid" gorm:"type:uuid;default:gen_random_uuid();uniqueIndex;not null"`
+ UserID uint `json:"user_id" gorm:"index;not null"`
+ WorkspaceID *uint `json:"workspace_id,omitempty" gorm:"index"`
+ DomainID uint `json:"domain_id" gorm:"index;not null"`
+ MessageID string `json:"message_id" gorm:"index"`
+ Sender string `json:"sender" gorm:"not null"`
+ Recipients pq.StringArray `json:"recipients" gorm:"type:text[];not null"`
+ Subject string `json:"subject"`
+ TextBody string `json:"text_body" gorm:"type:text"`
+ HTMLBody string `json:"html_body" gorm:"type:text"`
+ AttachmentsJSON string `json:"attachments_json,omitempty" gorm:"type:text"`
+ HeadersJSON string `json:"headers_json,omitempty" gorm:"type:text"`
+ RawStorageKey string `json:"raw_storage_key,omitempty"`
+ // RawContent holds the raw RFC 5322 message until the async
+ RawContent []byte `json:"-" gorm:"type:bytea"`
+ DedupHash string `json:"-" gorm:"index"`
+ Size int64 `json:"size"`
+ SpamScore *float64 `json:"spam_score,omitempty"`
+ Status InboundEmailStatus `json:"status" gorm:"default:received;not null"`
+ Source InboundSource `json:"source" gorm:"not null"`
+ ErrorMessage string `json:"error_message,omitempty"`
+ RetryCount int `json:"retry_count" gorm:"default:0;not null"`
+ ReceivedAt time.Time `json:"received_at" gorm:"not null"`
+ ForwardedAt *time.Time `json:"forwarded_at,omitempty"`
+ CreatedAt time.Time `json:"created_at"`
User User `json:"-" gorm:"foreignKey:UserID"`
Domain Domain `json:"-" gorm:"foreignKey:DomainID"`
@@ -76,6 +79,5 @@ type InboundAttachmentMeta struct {
ContentType string `json:"content_type"`
Size int64 `json:"size"`
StorageKey string `json:"storage_key,omitempty"`
- // Content is only populated when no blob store is configured (dev fallback).
- Content string `json:"content,omitempty"`
+ Content string `json:"content,omitempty"`
}
diff --git a/internal/services/inbound/service.go b/internal/services/inbound/service.go
index 7b06385..d2e4b39 100644
--- a/internal/services/inbound/service.go
+++ b/internal/services/inbound/service.go
@@ -24,6 +24,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"strings"
"time"
@@ -56,6 +57,7 @@ var (
// Enqueuer enqueues inbound-processing tasks. Satisfied by worker.Producer.
type Enqueuer interface {
EnqueueInboundProcess(inboundEmailID uint) error
+ EnqueueInboundParse(inboundEmailID uint) error
}
type Service struct {
@@ -274,6 +276,214 @@ func (s *Service) Ingest(ctx context.Context, p *ParsedEmail, source models.Inbo
return rec, nil
}
+func (s *Service) IngestRaw(ctx context.Context, raw []byte, envelopeFrom string, envelopeTo []string, source models.InboundSource) (*models.InboundEmail, error) {
+ start := time.Now()
+ defer func() {
+ if s.onIngestMs != nil {
+ s.onIngestMs(time.Since(start).Seconds())
+ }
+ }()
+
+ if s.maxMsgSize > 0 && int64(len(raw)) > s.maxMsgSize {
+ s.rejected("size_exceeded")
+ return nil, ErrSizeExceeded
+ }
+
+ domain, _, err := s.resolveDomain(envelopeTo)
+ if err != nil {
+ s.rejected("unverified_domain")
+ return nil, err
+ }
+
+ sender := strings.ToLower(strings.TrimSpace(envelopeFrom))
+ if s.suppressionRepo != nil && sender != "" {
+ scope := repositories.ResourceScope{UserID: domain.UserID, WorkspaceID: domain.WorkspaceID}
+ if yes, _ := s.suppressionRepo.IsSuppressed(scope, sender); yes {
+ s.rejected("sender_suppressed")
+ return s.persistRejectedRaw(raw, sender, envelopeTo, domain, source, "sender is on suppression list"), ErrSenderSuppressed
+ }
+ }
+
+ rec := &models.InboundEmail{
+ UserID: domain.UserID,
+ WorkspaceID: domain.WorkspaceID,
+ DomainID: domain.ID,
+ Sender: sender,
+ Recipients: envelopeTo,
+ Size: int64(len(raw)),
+ Status: models.InboundStatusReceived,
+ Source: source,
+ ReceivedAt: time.Now().UTC(),
+ RawContent: raw,
+ }
+ if err := s.repo.Create(rec); err != nil {
+ return nil, fmt.Errorf("persist inbound email: %w", err)
+ }
+
+ if s.blobStore != nil {
+ rawKey := fmt.Sprintf("inbound/%s/raw.eml", rec.UUID)
+ if err := s.blobStore.Put(ctx, rawKey, bytes.NewReader(raw), "message/rfc822"); err == nil {
+ rec.RawStorageKey = rawKey
+ rec.RawContent = nil
+ if uerr := s.repo.Update(rec); uerr != nil {
+ logger.Warn("failed to clear inline raw after blob upload", "uuid", rec.UUID, "error", uerr)
+ rec.RawContent = raw
+ }
+ } else {
+ logger.Warn("failed to store raw inbound eml", "uuid", rec.UUID, "error", err)
+ }
+ }
+
+ if s.producer != nil {
+ if err := s.producer.EnqueueInboundParse(rec.ID); err != nil {
+ logger.Error("failed to enqueue inbound:parse", "inbound_id", rec.ID, "error", err)
+ }
+ }
+
+ if s.onReceived != nil {
+ s.onReceived(source)
+ }
+ if s.onBytes != nil {
+ s.onBytes(rec.Size)
+ }
+ return rec, nil
+}
+
+func (s *Service) ApplyParsed(ctx context.Context, rec *models.InboundEmail, p *ParsedEmail) error {
+ if p.MessageID != "" {
+ if existing, err := s.repo.FindByMessageID(rec.UserID, p.MessageID); err == nil && existing != nil && existing.ID != rec.ID {
+ rec.MessageID = p.MessageID
+ rec.Status = models.InboundStatusRejected
+ rec.ErrorMessage = "duplicate message-id"
+ _ = s.repo.Update(rec)
+ return ErrDuplicate
+ } else if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+ logger.Warn("failed to check message-id dedup", "error", err)
+ }
+ }
+
+ recipients := rec.Recipients
+ if len(p.To) > 0 {
+ recipients = p.To
+ }
+
+ var dedupHash string
+ if p.MessageID == "" {
+ dedupHash = computeDedupHash(p.From, recipients, p.Subject, rec.Size)
+ if existing, err := s.repo.FindByDedupHash(rec.UserID, dedupHash); err == nil && existing != nil && existing.ID != rec.ID {
+ rec.DedupHash = dedupHash
+ rec.Status = models.InboundStatusRejected
+ rec.ErrorMessage = "duplicate content"
+ _ = s.repo.Update(rec)
+ return ErrDuplicate
+ } else if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+ logger.Warn("failed to check dedup hash", "error", err)
+ }
+ }
+
+ headersJSON, _ := json.Marshal(p.Headers)
+
+ rec.MessageID = p.MessageID
+ rec.DedupHash = dedupHash
+ if p.From != "" {
+ rec.Sender = p.From
+ }
+ if len(p.To) > 0 {
+ rec.Recipients = p.To
+ }
+ rec.Subject = p.Subject
+ rec.TextBody = p.TextBody
+ rec.HTMLBody = p.HTMLBody
+ rec.HeadersJSON = string(headersJSON)
+ rec.Status = models.InboundStatusReceived
+ rec.ErrorMessage = ""
+
+ attachments, uploadedKeys, err := s.persistAttachments(ctx, rec.UUID, p.Attachments)
+ if err != nil {
+ if s.blobStore != nil {
+ for _, k := range uploadedKeys {
+ _ = s.blobStore.Delete(ctx, k)
+ }
+ }
+ return fmt.Errorf("persist attachments: %w", err)
+ }
+ if len(attachments) > 0 {
+ if data, mErr := json.Marshal(attachments); mErr == nil {
+ rec.AttachmentsJSON = string(data)
+ }
+ }
+
+ if err := s.repo.Update(rec); err != nil {
+ return fmt.Errorf("update inbound email: %w", err)
+ }
+
+ if s.bus != nil {
+ actor := rec.UserID
+ s.bus.PublishSimple(
+ models.EventCategoryEmail,
+ "email.inbound.received",
+ &actor,
+ "",
+ "",
+ fmt.Sprintf("Inbound email received from %s", rec.Sender),
+ map[string]any{
+ "inbound_id": rec.UUID,
+ "sender": rec.Sender,
+ "recipients": []string(rec.Recipients),
+ "subject": rec.Subject,
+ "source": string(rec.Source),
+ "size": rec.Size,
+ "workspace_id": rec.WorkspaceID,
+ },
+ )
+ }
+ return nil
+}
+
+// LoadRaw returns the raw RFC 5322 bytes for a stored inbound record, reading
+// from the inline RawContent column first and falling back to the blob store
+// when the row has already been promoted.
+func (s *Service) LoadRaw(ctx context.Context, rec *models.InboundEmail) ([]byte, error) {
+ if len(rec.RawContent) > 0 {
+ return rec.RawContent, nil
+ }
+ if rec.RawStorageKey == "" || s.blobStore == nil {
+ return nil, fmt.Errorf("raw content unavailable for inbound %s", rec.UUID)
+ }
+ rc, err := s.blobStore.Get(ctx, rec.RawStorageKey)
+ if err != nil {
+ return nil, fmt.Errorf("fetch raw blob: %w", err)
+ }
+ defer func() { _ = rc.Close() }()
+ raw, err := io.ReadAll(rc)
+ if err != nil {
+ return nil, fmt.Errorf("read raw blob: %w", err)
+ }
+ return raw, nil
+}
+
+// persistRejectedRaw records a synchronous-time rejection from IngestRaw where
+// only envelope information is known — no parsed headers, no subject.
+func (s *Service) persistRejectedRaw(raw []byte, sender string, recipients []string, domain *models.Domain, source models.InboundSource, reason string) *models.InboundEmail {
+ rec := &models.InboundEmail{
+ UserID: domain.UserID,
+ WorkspaceID: domain.WorkspaceID,
+ DomainID: domain.ID,
+ Sender: sender,
+ Recipients: recipients,
+ Size: int64(len(raw)),
+ Status: models.InboundStatusRejected,
+ Source: source,
+ ReceivedAt: time.Now().UTC(),
+ ErrorMessage: reason,
+ }
+ if err := s.repo.Create(rec); err != nil {
+ logger.Error("failed to persist rejected inbound email", "error", err)
+ return nil
+ }
+ return rec
+}
+
// resolveDomain walks the recipient list and returns the first ownership-verified
// tenant domain that matches. Returns ErrUnverifiedDomain if none match.
func (s *Service) resolveDomain(recipients []string) (*models.Domain, string, error) {
diff --git a/internal/services/inbound/smtp_server.go b/internal/services/inbound/smtp_server.go
index 09ed3cc..aaacd51 100644
--- a/internal/services/inbound/smtp_server.go
+++ b/internal/services/inbound/smtp_server.go
@@ -121,8 +121,11 @@ func (s *session) Rcpt(to string, _ *smtp.RcptOptions) error {
return nil
}
-// Data reads the raw message (bounded by MaxMessageSize), parses it, and delegates
-// to the service. Maps typed service errors back to SMTP status codes.
+// Data reads the raw message (bounded by MaxMessageSize) and hands the bytes
+// to the store-first ingest path. Parsing is deferred to the async worker,
+// so a malformed MIME body or a Postgres encoding error during structured
+// persistence no longer maps to 451 — once IngestRaw succeeds, the message
+// is durably stored and the MTA gets 250.
func (s *session) Data(r io.Reader) error {
limit := s.backend.maxMessageSize
if limit <= 0 {
@@ -138,28 +141,17 @@ func (s *session) Data(r io.Reader) error {
return &smtp.SMTPError{Code: 552, EnhancedCode: smtp.EnhancedCode{5, 3, 4}, Message: "message size exceeds limit"}
}
- parsed, err := ParseRawEmail(raw)
- if err != nil {
- logger.Warn("smtp parse error", "remote", s.remote, "error", err)
- return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 6, 0}, Message: "malformed message"}
- }
- if parsed.From == "" && s.from != "" {
- parsed.From = s.from
- }
- if len(parsed.To) == 0 {
- parsed.To = append([]string(nil), s.to...)
- }
-
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
- _, ierr := s.backend.svc.Ingest(ctx, parsed, models.InboundSourceSMTP)
+ recipients := append([]string(nil), s.to...)
+ _, ierr := s.backend.svc.IngestRaw(ctx, raw, s.from, recipients, models.InboundSourceSMTP)
switch {
- case ierr == nil, errors.Is(ierr, ErrDuplicate):
+ case ierr == nil:
return nil
case errors.Is(ierr, ErrUnverifiedDomain):
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 1, 1}, Message: "relay not permitted"}
- case errors.Is(ierr, ErrSizeExceeded), errors.Is(ierr, ErrAttachmentTooLarge):
+ case errors.Is(ierr, ErrSizeExceeded):
return &smtp.SMTPError{Code: 552, EnhancedCode: smtp.EnhancedCode{5, 3, 4}, Message: "message too large"}
case errors.Is(ierr, ErrSenderSuppressed):
return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.EnhancedCode{5, 7, 1}, Message: "sender suppressed"}
diff --git a/internal/worker/inbound_handler.go b/internal/worker/inbound_handler.go
index 7ea5691..6ca1070 100644
--- a/internal/worker/inbound_handler.go
+++ b/internal/worker/inbound_handler.go
@@ -23,12 +23,14 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/json"
+ "errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/goposta/posta/internal/models"
+ "github.com/goposta/posta/internal/services/inbound"
"github.com/goposta/posta/internal/services/webhook"
"github.com/goposta/posta/internal/storage/repositories"
"github.com/hibiken/asynq"
@@ -62,6 +64,71 @@ type InboundWebhookPayload struct {
ReceivedAt string `json:"received_at"`
}
+type InboundParseHandler struct {
+ repo *repositories.InboundEmailRepository
+ svc *inbound.Service
+ producer *Producer
+ onParsed func()
+}
+
+func NewInboundParseHandler(repo *repositories.InboundEmailRepository, svc *inbound.Service, producer *Producer) *InboundParseHandler {
+ return &InboundParseHandler{repo: repo, svc: svc, producer: producer}
+}
+
+// OnParsed sets a callback invoked after each successfully parsed record.
+func (h *InboundParseHandler) OnParsed(fn func()) { h.onParsed = fn }
+
+func (h *InboundParseHandler) ProcessTask(ctx context.Context, t *asynq.Task) error {
+ var payload InboundParsePayload
+ if err := json.Unmarshal(t.Payload(), &payload); err != nil {
+ return fmt.Errorf("unmarshal inbound parse payload: %w: %w", err, asynq.SkipRetry)
+ }
+
+ rec, err := h.repo.FindByID(payload.InboundEmailID)
+ if err != nil {
+ return fmt.Errorf("inbound email not found: %w", err)
+ }
+
+ // Defensive: another retry already parsed and moved this record on. Skip.
+ if rec.Status != models.InboundStatusReceived {
+ return nil
+ }
+
+ raw, err := h.svc.LoadRaw(ctx, rec)
+ if err != nil {
+ return fmt.Errorf("load raw inbound bytes: %w", err)
+ }
+
+ parsed, perr := inbound.ParseRawEmail(raw)
+ if perr != nil {
+ rec.Status = models.InboundStatusQuarantined
+ rec.ErrorMessage = fmt.Sprintf("parse failed: %v", perr)
+ if uerr := h.repo.Update(rec); uerr != nil {
+ logger.Error("failed to mark inbound quarantined after parse error", "id", rec.ID, "error", uerr)
+ }
+ logger.Warn("inbound parse error, record quarantined", "id", rec.ID, "uuid", rec.UUID, "error", perr)
+ return nil
+ }
+
+ if aerr := h.svc.ApplyParsed(ctx, rec, parsed); aerr != nil {
+ if errors.Is(aerr, inbound.ErrDuplicate) {
+ return nil
+ }
+ return fmt.Errorf("apply parsed inbound: %w", aerr)
+ }
+
+ if h.producer != nil {
+ if err := h.producer.EnqueueInboundProcess(rec.ID); err != nil {
+ logger.Error("failed to enqueue inbound:process", "inbound_id", rec.ID, "error", err)
+ }
+ }
+
+ if h.onParsed != nil {
+ h.onParsed()
+ }
+ return nil
+}
+
// InboundProcessHandler processes inbound:process tasks — builds the inbound
// webhook payload and dispatches it via the webhook dispatcher.
type InboundProcessHandler struct {
@@ -210,23 +277,46 @@ func ChainErrorHandlers(handlers ...asynq.ErrorHandler) asynq.ErrorHandler {
}
func (e *InboundExhaustedErrorHandler) HandleError(_ context.Context, t *asynq.Task, err error) {
- if t.Type() != TypeInboundProcess {
- return
- }
- var payload InboundProcessPayload
- if jerr := json.Unmarshal(t.Payload(), &payload); jerr != nil {
- logger.Error("inbound exhausted: unmarshal", "error", jerr)
+ var (
+ id uint
+ finalStatus models.InboundEmailStatus
+ logMsg string
+ )
+ switch t.Type() {
+ case TypeInboundProcess:
+ var payload InboundProcessPayload
+ if jerr := json.Unmarshal(t.Payload(), &payload); jerr != nil {
+ logger.Error("inbound exhausted: unmarshal", "error", jerr)
+ return
+ }
+ id = payload.InboundEmailID
+ finalStatus = models.InboundStatusFailed
+ logMsg = "worker: inbound forward permanently failed"
+ case TypeInboundParse:
+ var payload InboundParsePayload
+ if jerr := json.Unmarshal(t.Payload(), &payload); jerr != nil {
+ logger.Error("inbound parse exhausted: unmarshal", "error", jerr)
+ return
+ }
+ id = payload.InboundEmailID
+ // Parse-time exhaustion lands in quarantine, not failed: the raw bytes
+ // are still durable and an operator can retry once the underlying
+ // cause (DB outage, blob fetch, malformed message) is resolved.
+ finalStatus = models.InboundStatusQuarantined
+ logMsg = "worker: inbound parse permanently failed, quarantined"
+ default:
return
}
- rec, ferr := e.repo.FindByID(payload.InboundEmailID)
+
+ rec, ferr := e.repo.FindByID(id)
if ferr != nil {
return
}
- rec.Status = models.InboundStatusFailed
+ rec.Status = finalStatus
rec.ErrorMessage = fmt.Sprintf("permanently failed after retries: %v", err)
_ = e.repo.Update(rec)
if e.onFailed != nil {
e.onFailed()
}
- logger.Error("worker: inbound forward permanently failed", "id", rec.ID, "error", err)
+ logger.Error(logMsg, "id", rec.ID, "error", err)
}
diff --git a/internal/worker/producer.go b/internal/worker/producer.go
index 1d33d03..0c45d8e 100644
--- a/internal/worker/producer.go
+++ b/internal/worker/producer.go
@@ -141,6 +141,24 @@ func (p *Producer) EnqueueInboundProcess(inboundEmailID uint) error {
return nil
}
+// EnqueueInboundParse enqueues a parse task for a raw inbound message that
+// was just persisted by the SMTP receiver. The parse step is intentionally
+// separate from EnqueueInboundProcess so a permanent parse failure does not
+// block the SMTP 250 OK reply.
+func (p *Producer) EnqueueInboundParse(inboundEmailID uint) error {
+ task, err := NewInboundParseTask(inboundEmailID,
+ asynq.Queue(QueueTransactional),
+ asynq.MaxRetry(p.maxRetries),
+ )
+ if err != nil {
+ return fmt.Errorf("failed to create inbound parse task: %w", err)
+ }
+ if _, err := p.client.Enqueue(task); err != nil {
+ return fmt.Errorf("failed to enqueue inbound parse task: %w", err)
+ }
+ return nil
+}
+
// Close closes the underlying Asynq client.
func (p *Producer) Close() error {
return p.client.Close()
diff --git a/internal/worker/tasks.go b/internal/worker/tasks.go
index e24451f..9b495c2 100644
--- a/internal/worker/tasks.go
+++ b/internal/worker/tasks.go
@@ -27,6 +27,7 @@ const (
TypeEmailSend = "email:send"
TypeCampaignStart = "campaign:start"
TypeCampaignBatch = "campaign:batch"
+ TypeInboundParse = "inbound:parse"
TypeInboundProcess = "inbound:process"
QueueTransactional = "transactional"
@@ -80,3 +81,17 @@ func NewInboundProcessTask(id uint, opts ...asynq.Option) (*asynq.Task, error) {
}
return asynq.NewTask(TypeInboundProcess, payload, opts...), nil
}
+
+type InboundParsePayload struct {
+ InboundEmailID uint `json:"inbound_email_id"`
+}
+
+// NewInboundParseTask creates an Asynq task that parses a previously-received
+// raw inbound message into its structured form.
+func NewInboundParseTask(id uint, opts ...asynq.Option) (*asynq.Task, error) {
+ payload, err := json.Marshal(InboundParsePayload{InboundEmailID: id})
+ if err != nil {
+ return nil, err
+ }
+ return asynq.NewTask(TypeInboundParse, payload, opts...), nil
+}
diff --git a/web/src/api/types.ts b/web/src/api/types.ts
index 9ee5b2e..8baf25e 100644
--- a/web/src/api/types.ts
+++ b/web/src/api/types.ts
@@ -71,7 +71,7 @@ export interface Email {
provider?: string
}
-export type InboundEmailStatus = 'received' | 'forwarded' | 'failed' | 'rejected'
+export type InboundEmailStatus = 'received' | 'forwarded' | 'failed' | 'rejected' | 'quarantined'
export type InboundSource = 'smtp' | 'webhook'
export interface InboundAttachmentMeta {
diff --git a/web/src/views/inbound/InboundEmailDetail.vue b/web/src/views/inbound/InboundEmailDetail.vue
index b838465..0d3a9bd 100644
--- a/web/src/views/inbound/InboundEmailDetail.vue
+++ b/web/src/views/inbound/InboundEmailDetail.vue
@@ -36,12 +36,17 @@ async function load() {
async function retryEmail() {
if (!email.value || retrying.value) return
+ const wasQuarantined = email.value.status === 'quarantined'
retrying.value = true
try {
const res = await inboundApi.retry(email.value.uuid)
email.value.status = res.data.data.status as InboundEmail['status']
email.value.error_message = ''
- notify.success('Inbound email re-queued for webhook dispatch')
+ notify.success(
+ wasQuarantined
+ ? 'Inbound email re-queued for parsing'
+ : 'Inbound email re-queued for webhook dispatch'
+ )
} catch (e: any) {
const msg = e.response?.data?.error?.message || 'Failed to retry inbound email'
notify.error(msg)
@@ -86,6 +91,7 @@ function statusBadgeClass(status: string) {
switch (status) {
case 'forwarded': return 'badge badge-success'
case 'failed': return 'badge badge-danger'
+ case 'quarantined': return 'badge badge-danger'
case 'received': return 'badge badge-info'
case 'rejected': return 'badge badge-warning'
default: return 'badge'
@@ -115,12 +121,12 @@ function formatBytes(n: number) {
<h1>Inbound Email</h1>
<div style="display: flex; gap: 8px">
<button
- v-if="email && (email.status === 'failed' || email.status === 'received')"
+ v-if="email && (email.status === 'failed' || email.status === 'received' || email.status === 'quarantined')"
class="btn btn-primary"
:disabled="retrying"
@click="retryEmail"
>
- {{ retrying ? 'Retrying...' : 'Retry dispatch' }}
+ {{ retrying ? 'Retrying...' : (email.status === 'quarantined' ? 'Retry parse' : 'Retry dispatch') }}
</button>
<a
v-if="hasRawEml && email"
diff --git a/web/src/views/inbound/InboundEmails.vue b/web/src/views/inbound/InboundEmails.vue
index 9093398..038ec2f 100644
--- a/web/src/views/inbound/InboundEmails.vue
+++ b/web/src/views/inbound/InboundEmails.vue
@@ -77,6 +77,8 @@ function statusBadgeClass(status: string) {
return "badge badge-success";
case "failed":
return "badge badge-danger";
+ case "quarantined":
+ return "badge badge-danger";
case "received":
return "badge badge-info";
case "rejected":
@@ -136,6 +138,7 @@ function formatBytes(n: number) {
<option value="received">Received</option>
<option value="forwarded">Forwarded</option>
<option value="failed">Failed</option>
+ <option value="quarantined">Quarantined</option>
<option value="rejected">Rejected</option>
</select>
</div>