The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
package outbox
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/tracewayapp/traceway/backend/app/config"
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/repositories/transactional"
traceway "go.tracewayapp.com"
)
const (
drainAdvisoryLockId = 824737003 // escalator: 824737002, backfill: 824737001
drainBatchSize = 50
drainConcurrency = 8
sendTimeout = 30 * time.Second
sendAbandonGrace = 5 * time.Second
staleSendingAge = 5 * time.Minute
maxAttempts = 5
)
// backoffSchedule[attempts-1] = delay before the next attempt (attempts was
// already incremented at claim time). attempts >= maxAttempts is terminal.
var backoffSchedule = []time.Duration{1 * time.Minute, 5 * time.Minute, 15 * time.Minute, 60 * time.Minute}
var (
sentTotal uint64
terminalFailuresTotal uint64
)
func drainPollInterval() time.Duration {
return config.PollSeconds(config.Config.OutboxPollSeconds, 15)
}
// StartDrain runs the outbox drain loop: claim due rows in a transaction
// (marking them sending), deliver after commit, finalize each row in its own
// small transaction. All state lives in notification_outbox, so a crash at any
// point is recovered: unclaimed rows stay due, stale sending rows are
// reclaimed, and the guarded status transitions make cancellation win races.
func StartDrain(ctx context.Context) {
go func() {
defer traceway.Recover()
ticker := time.NewTicker(drainPollInterval())
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
case <-wakeCh:
}
DrainOnce(ctx, time.Now().UTC())
}
}()
}
// DrainOnce runs a single drain tick: reclaim stale sending rows, claim due
// rows, deliver them, finalize. Exported so tests (and tooling) can drive the
// outbox deterministically.
func DrainOnce(ctx context.Context, now time.Time) {
dueCount := 0
claimed, err := db.ExecuteTransaction(func(tx *sql.Tx) ([]*models.OutboxDelivery, error) {
if !db.IsSQLite() {
if _, err := tx.Exec(fmt.Sprintf("SELECT pg_advisory_xact_lock(%d)", drainAdvisoryLockId)); err != nil {
return nil, fmt.Errorf("failed to take outbox drain lock: %w", err)
}
}
if err := transactional.OutboxRepository.ReclaimStaleSending(tx, now.Add(-staleSendingAge), now); err != nil {
return nil, err
}
due, err := transactional.OutboxRepository.FindDue(tx, now, drainBatchSize)
if err != nil {
return nil, err
}
dueCount = len(due)
claimed := make([]*models.OutboxDelivery, 0, len(due))
for _, row := range due {
won, err := transactional.OutboxRepository.MarkSending(tx, row.Id, now)
if err != nil {
return nil, err
}
if !won {
// Cancelled (acknowledge/resolve) since the due list was read;
// the row must not be sent.
continue
}
row.Attempts++
claimed = append(claimed, row)
}
return claimed, nil
})
if err != nil {
traceway.CaptureException(fmt.Errorf("outbox drain tick failed: %w", err))
return
}
var wg sync.WaitGroup
sem := make(chan struct{}, drainConcurrency)
for _, row := range claimed {
wg.Add(1)
sem <- struct{}{}
go func(row *models.OutboxDelivery) {
defer traceway.Recover()
defer wg.Done()
defer func() { <-sem }()
sendRow(ctx, row)
}(row)
}
wg.Wait()
if dueCount == drainBatchSize {
// A full batch means more rows may already be due.
Wake()
}
}
func sendRow(ctx context.Context, row *models.OutboxDelivery) {
if sender == nil {
finalizeRow(row, errors.New("outbox sender not registered"))
return
}
var msg models.NotificationMessage
if err := json.Unmarshal(row.Message, &msg); err != nil {
// Poison row: retrying cannot fix an unreadable payload.
finalizeTerminal(row, "unreadable message payload: "+err.Error())
return
}
finalizeRow(row, sendWithDeadline(ctx, row, msg))
}
// sendWithDeadline bounds one attempt in wall-clock time; an adapter that
// ignores its context deadline is abandoned and the row retries (at-least-once).
func sendWithDeadline(ctx context.Context, row *models.OutboxDelivery, msg models.NotificationMessage) error {
sendCtx, cancel := context.WithTimeout(ctx, sendTimeout)
defer cancel()
done := make(chan error, 1)
go func() {
defer func() {
if r := recover(); r != nil {
done <- fmt.Errorf("adapter %q panicked: %v", row.AdapterType, r)
}
}()
done <- sender(sendCtx, row.AdapterType, json.RawMessage(row.AdapterConfig), msg)
}()
abandonAfter := time.NewTimer(sendTimeout + sendAbandonGrace)
defer abandonAfter.Stop()
select {
case err := <-done:
return err
case <-abandonAfter.C:
return fmt.Errorf("adapter %q ignored its %s deadline and was abandoned", row.AdapterType, sendTimeout)
}
}
func finalizeRow(row *models.OutboxDelivery, sendErr error) {
if sendErr != nil && row.Attempts >= maxAttempts {
finalizeTerminal(row, sendErr.Error())
return
}
now := time.Now().UTC()
finalized, err := db.ExecuteTransaction(func(tx *sql.Tx) (bool, error) {
if sendErr == nil {
sent, err := transactional.OutboxRepository.MarkSent(tx, row.Id, now)
if err != nil {
return false, err
}
if !sent {
// Cancelled while the send was in flight (the documented
// at-least-once window): the cancel already mirrored the
// page_notifications row, which must not be rewritten.
return false, nil
}
if row.PageNotificationId != nil {
if err := transactional.PageNotificationRepository.MarkSent(tx, *row.PageNotificationId, now); err != nil {
return false, err
}
}
return true, nil
}
next := now.Add(backoffSchedule[backoffIndex(row.Attempts)])
return transactional.OutboxRepository.MarkFailedWithBackoff(tx, row.Id, sendErr.Error(), &next, now)
})
if err != nil {
traceway.CaptureException(fmt.Errorf("failed to finalize outbox row %d: %w", row.Id, err))
return
}
if sendErr != nil {
// Retryable failures must be visible before they turn terminal.
traceway.CaptureException(fmt.Errorf("outbox delivery %d (%s via %s) attempt %d failed, will retry: %w", row.Id, row.Kind, row.AdapterType, row.Attempts, sendErr))
return
}
if finalized {
atomic.AddUint64(&sentTotal, 1)
if terminalHook != nil {
terminalHook(row, models.OutboxSent, "")
}
}
}
func finalizeTerminal(row *models.OutboxDelivery, errorMsg string) {
now := time.Now().UTC()
finalized, err := db.ExecuteTransaction(func(tx *sql.Tx) (bool, error) {
failed, err := transactional.OutboxRepository.MarkFailedWithBackoff(tx, row.Id, errorMsg, nil, now)
if err != nil {
return false, err
}
if !failed {
// Cancelled while the send was in flight; the cancel already
// mirrored the page_notifications row.
return false, nil
}
if row.PageNotificationId != nil {
if err := transactional.PageNotificationRepository.MarkFailed(tx, *row.PageNotificationId, errorMsg, now); err != nil {
return false, err
}
}
return true, nil
})
if err != nil {
traceway.CaptureException(fmt.Errorf("failed to finalize outbox row %d: %w", row.Id, err))
return
}
if !finalized {
return
}
atomic.AddUint64(&terminalFailuresTotal, 1)
traceway.CaptureException(fmt.Errorf("outbox delivery %d (%s via %s) permanently failed after %d attempts: %s", row.Id, row.Kind, row.AdapterType, row.Attempts, errorMsg))
if terminalHook != nil {
terminalHook(row, models.OutboxFailed, errorMsg)
}
}
func backoffIndex(attempts int) int {
index := attempts - 1
if index < 0 {
index = 0
}
if index >= len(backoffSchedule) {
index = len(backoffSchedule) - 1
}
return index
}