Skip to content

The only tool you need to know what is happening and how to fix it.

19.3 KB
Raw Blame
package oncall

import (
	"context"
	"crypto/rand"
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/tracewayapp/traceway/backend/app/config"
	"github.com/tracewayapp/traceway/backend/app/db"
	"github.com/tracewayapp/traceway/backend/app/models"
	"github.com/tracewayapp/traceway/backend/app/notifications"
	"github.com/tracewayapp/traceway/backend/app/outbox"
	"github.com/tracewayapp/traceway/backend/app/repositories/transactional"
	"github.com/tracewayapp/traceway/backend/app/repositories/transactional/shared"
	traceway "go.tracewayapp.com"
)

const escalatorAdvisoryLockId = 824737002

// errClaimLost aborts a claim transaction when the guarded escalation-state
// update matches no row: a concurrent acknowledge/resolve committed after the
// due check, and its CancelByKey could not see this claim's uncommitted
// deliveries — so they must never commit. Rolling back is the normal loss of
// a benign race, not an error worth reporting.
var errClaimLost = errors.New("page claim lost to a concurrent acknowledge/resolve")

var wakeCh = make(chan struct{}, 1)

// Wake nudges the escalator so a freshly opened page notifies its first level
// immediately instead of waiting for the next tick. Non-blocking.
func Wake() {
	select {
	case wakeCh <- struct{}{}:
	default:
	}
}

func escalatorPollInterval() time.Duration {
	return config.PollSeconds(config.Config.OncallPollSeconds, 30)
}

// StartEscalator runs the ack-based escalation loop. Each tick is purely
// transactional: due pages advance a level and their deliveries are enqueued
// into the notification outbox, which owns sending, retries, and crash
// recovery. Ack/resolve cancels queued deliveries via outbox.CancelByKey.
func StartEscalator(ctx context.Context) {
	go func() {
		defer traceway.Recover()

		ticker := time.NewTicker(escalatorPollInterval())
		defer ticker.Stop()
		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
			case <-wakeCh:
			}
			runEscalatorTick(ctx, time.Now().UTC())
		}
	}()
}

func runEscalatorTick(_ context.Context, now time.Time) {
	// Each page is claimed in its own transaction so one failing page (e.g. a
	// target schedule whose stored definition no longer parses) cannot roll
	// back or block escalation for every other page.
	pageIds, err := db.ExecuteTransaction(func(tx *sql.Tx) ([]int, error) {
		duePages, err := transactional.PageRepository.FindDue(tx, now)
		if err != nil {
			return nil, err
		}
		ids := make([]int, 0, len(duePages))
		for _, page := range duePages {
			ids = append(ids, page.Id)
		}
		return ids, nil
	})
	if err != nil {
		traceway.CaptureException(fmt.Errorf("escalator tick failed: %w", err))
		return
	}

	enqueued := 0
	for _, pageId := range pageIds {
		count, err := db.ExecuteTransaction(func(tx *sql.Tx) (int, error) {
			if !db.IsSQLite() {
				if _, err := tx.Exec(fmt.Sprintf("SELECT pg_advisory_xact_lock(%d)", escalatorAdvisoryLockId)); err != nil {
					return 0, fmt.Errorf("failed to take escalator lock: %w", err)
				}
			}
			page, err := transactional.PageRepository.FindDueById(tx, pageId, now)
			if err != nil {
				return 0, err
			}
			if page == nil {
				// Acknowledged, resolved, or claimed by a concurrent escalator
				// since the due list was read.
				return 0, nil
			}
			return claimPageEscalation(tx, page, now)
		})
		if err != nil {
			if errors.Is(err, errClaimLost) {
				continue
			}
			traceway.CaptureException(fmt.Errorf("failed to escalate page %d: %w", pageId, err))
			continue
		}
		enqueued += count
	}
	if enqueued > 0 {
		outbox.Wake()
	}
}

// claimPageEscalation advances one due page to its next escalation level and
// enqueues its deliveries. Returns the number of outbox rows created. Runs
// inside the claim transaction.
func claimPageEscalation(tx *sql.Tx, page *models.Page, now time.Time) (int, error) {
	definition, err := ParsePolicyDefinition(page.PolicySnapshot)
	if err != nil || len(definition.Steps) == 0 {
		traceway.CaptureException(fmt.Errorf("page %d has an unusable policy snapshot, stopping its escalation", page.Id))
		return 0, updateEscalationState(tx, page.Id, page.EscalationLevel, page.RepeatIteration, nil, now)
	}

	level := page.EscalationLevel + 1
	if level >= len(definition.Steps) {
		return 0, updateEscalationState(tx, page.Id, page.EscalationLevel, page.RepeatIteration, nil, now)
	}
	step := definition.Steps[level]

	enqueued := 0
	notifiedUsers := map[int]bool{}
	for _, target := range step.Targets {
		switch target.Type {
		case TargetSchedule, TargetTeam, TargetUser:
			userIds, err := resolveUserTarget(tx, page.OrganizationId, target, now)
			if err != nil {
				return 0, err
			}
			for _, userId := range userIds {
				if notifiedUsers[userId] {
					continue
				}
				notifiedUsers[userId] = true
				count, err := claimUserDeliveries(tx, page, level, userId, now)
				if err != nil {
					return 0, err
				}
				enqueued += count
			}
		case TargetChannel:
			count, err := claimChannelDelivery(tx, page, level, target.Id, now)
			if err != nil {
				return 0, err
			}
			enqueued += count
		default:
			traceway.CaptureException(fmt.Errorf("page %d step %d has unknown target type %q", page.Id, level, target.Type))
		}
	}

	lastStep := level == len(definition.Steps)-1
	switch {
	case !lastStep:
		next := now.Add(time.Duration(step.DelayMinutes) * time.Minute)
		err = updateEscalationState(tx, page.Id, level, page.RepeatIteration, &next, now)
	case page.RepeatIteration < definition.RepeatCount:
		next := now.Add(time.Duration(step.DelayMinutes) * time.Minute)
		err = updateEscalationState(tx, page.Id, -1, page.RepeatIteration+1, &next, now)
	default:
		// Exhausted: the page stays open until someone acknowledges or
		// resolves it, but nothing further is sent.
		err = updateEscalationState(tx, page.Id, level, page.RepeatIteration, nil, now)
	}
	return enqueued, err
}

// updateEscalationState is the claim path's terminal write: every branch of a
// claim must go through the status-guarded update so a page acknowledged or
// resolved mid-claim rolls the whole claim back via errClaimLost.
func updateEscalationState(tx *sql.Tx, pageId int, level int, iteration int, nextEscalationAt *time.Time, now time.Time) error {
	updated, err := transactional.PageRepository.UpdateEscalationState(tx, pageId, level, iteration, nextEscalationAt, now)
	if err != nil {
		return err
	}
	if !updated {
		return errClaimLost
	}
	return nil
}

// resolveUserTarget maps a schedule/team/user target to concrete user ids.
// Dangling targets resolve to nothing (reported, never fatal).
func resolveUserTarget(tx *sql.Tx, organizationId int, target StepTarget, now time.Time) ([]int, error) {
	switch target.Type {
	case TargetSchedule:
		userIds, err := CurrentOnCallForSchedule(tx, target.Id, now)
		if err != nil {
			return nil, err
		}
		if len(userIds) == 0 {
			traceway.CaptureException(fmt.Errorf("escalation target schedule %d resolved to nobody on call", target.Id))
		}
		return userIds, nil
	case TargetTeam:
		team, err := transactional.TeamRepository.FindById(tx, target.Id)
		if err != nil {
			return nil, err
		}
		if team == nil || team.OrganizationId != organizationId {
			traceway.CaptureException(fmt.Errorf("escalation target team %d no longer exists", target.Id))
			return nil, nil
		}
		return transactional.TeamRepository.FindMemberUserIds(tx, team.Id)
	case TargetUser:
		role, err := transactional.OrganizationRepository.GetUserRole(tx, organizationId, target.Id)
		if err != nil {
			return nil, err
		}
		if role == "" {
			traceway.CaptureException(fmt.Errorf("escalation target user %d is no longer an organization member", target.Id))
			return nil, nil
		}
		return []int{target.Id}, nil
	}
	return nil, nil
}

// claimUserDeliveries runs the user's notification-rule chain for the page's
// urgency: every step is enqueued at once as a scheduled outbox delivery
// (NotBefore staggered by the step delay) under the page's cancel key, so
// acknowledging cancels the tail with no separate scheduler. Users without a
// usable chain fall back to every enabled+verified method immediately, or the
// account email when none exist. Each delivery row mints its own ack token.
func claimUserDeliveries(tx *sql.Tx, page *models.Page, level int, userId int, now time.Time) (int, error) {
	user, err := transactional.UserRepository.FindById(tx, userId)
	if err != nil {
		return 0, err
	}
	if user == nil {
		return 0, nil
	}
	methods, err := transactional.UserContactMethodRepository.FindEnabledByUser(tx, userId)
	if err != nil {
		return 0, err
	}
	// Dropping unsendable methods here (rather than at delivery time) keeps the
	// "no methods left" account-email fallback below reachable: a user whose
	// only method is an SMS row from when Twilio was still configured must
	// still be paged.
	methods = sendableContactMethods(methods)
	methodById := make(map[int]*models.UserContactMethod, len(methods))
	for _, method := range methods {
		methodById[method.Id] = method
	}

	enqueued := 0
	appendDelivery := func(methodType string, configJSON json.RawMessage, desc string, delay time.Duration) error {
		token := newAckToken()
		sendAt := now.Add(delay)
		notification := &models.PageNotification{
			PageId:       page.Id,
			Level:        level,
			Iteration:    page.RepeatIteration,
			UserId:       &userId,
			TargetDesc:   truncateTargetDesc(desc),
			MethodType:   methodType,
			Status:       models.PageNotificationPending,
			ScheduledFor: &sendAt,
			AckTokenHash: shared.HashAuthToken(token),
			CreatedAt:    now,
		}
		notificationId, err := transactional.PageNotificationRepository.Create(tx, notification)
		if err != nil {
			return err
		}
		var notBefore *time.Time
		if delay > 0 {
			notBefore = &sendAt
		}
		if _, err := outbox.Enqueue(tx, outbox.Delivery{
			Kind:               models.OutboxKindPage,
			AdapterType:        methodType,
			AdapterConfig:      configJSON,
			Message:            buildPageMessage(page, level, ackURLFor(token)),
			NotBefore:          notBefore,
			CancelKey:          outbox.PageCancelKey(page.Id),
			PageNotificationId: &notificationId,
		}); err != nil {
			return err
		}
		enqueued++
		return nil
	}

	deliveryFor := func(method *models.UserContactMethod) (json.RawMessage, string, bool) {
		switch method.MethodType {
		case "email":
			configJSON, desc := EmailDeliveryFor(user.Email, ParseEmailOverride(method.Config))
			return configJSON, desc, true
		case "sms":
			return json.RawMessage(method.Config), smsDescFor(method.Config), true
		case "slack", "pushover", "telegram":
			return json.RawMessage(method.Config), user.Name + " (" + method.MethodType + ")", true
		default:
			traceway.CaptureException(fmt.Errorf("user %d has a contact method of unsupported type %q", userId, method.MethodType))
			return nil, "", false
		}
	}

	urgency := page.Urgency
	if urgency == "" {
		urgency = ResolveUrgency("", page.Severity)
	}
	rules, err := transactional.UserNotificationRuleRepository.FindByUserAndUrgency(tx, userId, urgency)
	if err != nil {
		return 0, err
	}
	type chainStep struct {
		methodType string
		configJSON json.RawMessage
		desc       string
		delay      time.Duration
	}
	var chain []chainStep
	droppedBeforeFirstStep := false
	for _, rule := range rules {
		method, ok := methodById[rule.ContactMethodId]
		if ok {
			configJSON, desc, sendable := deliveryFor(method)
			if sendable {
				chain = append(chain, chainStep{
					methodType: method.MethodType,
					configJSON: configJSON,
					desc:       desc,
					delay:      time.Duration(rule.DelayMinutes) * time.Minute,
				})
				continue
			}
		}
		if len(chain) == 0 {
			droppedBeforeFirstStep = true
		}
	}
	if len(chain) > 0 {
		// Rebase so the earliest surviving step still pages immediately when
		// the step carrying the chain's zero delay was dropped.
		offset := time.Duration(0)
		if droppedBeforeFirstStep {
			offset = chain[0].delay
		}
		for _, step := range chain {
			delay := step.delay - offset
			desc := step.desc
			if delay > 0 {
				desc = fmt.Sprintf("%s, +%dm", desc, int(delay/time.Minute))
			}
			if err := appendDelivery(step.methodType, step.configJSON, desc, delay); err != nil {
				return 0, err
			}
		}
		return enqueued, nil
	}
	if len(rules) > 0 {
		traceway.CaptureException(fmt.Errorf("user %d has %s-urgency notification rules but no usable steps, falling back", userId, urgency))
	}

	if len(methods) == 0 {
		// Email is never silently missing: no configured methods means the
		// account email is paged.
		configJSON, desc := EmailDeliveryFor(user.Email, "")
		if err := appendDelivery("email", configJSON, desc, 0); err != nil {
			return 0, err
		}
		return enqueued, nil
	}
	for _, method := range methods {
		configJSON, desc, ok := deliveryFor(method)
		if !ok {
			continue
		}
		if err := appendDelivery(method.MethodType, configJSON, desc, 0); err != nil {
			return 0, err
		}
	}
	return enqueued, nil
}

// sendableContactMethods drops methods this instance has no transport for.
// Only SMS can lose its transport: Twilio credentials can be removed after the
// method was created, and paging into a dead channel is worse than falling
// back to the account email.
func sendableContactMethods(methods []*models.UserContactMethod) []*models.UserContactMethod {
	if config.Config.TwilioEnabled() {
		return methods
	}
	sendable := make([]*models.UserContactMethod, 0, len(methods))
	for _, method := range methods {
		if method.MethodType == "sms" {
			continue
		}
		sendable = append(sendable, method)
	}
	return sendable
}

// maxTargetDescLength matches page_notifications.target_desc (VARCHAR(300) on
// Postgres).
const maxTargetDescLength = 300

// truncateTargetDesc clamps a delivery-log label to the column limit; an
// oversized label would fail the insert and roll back the escalation claim.
func truncateTargetDesc(desc string) string {
	if len(desc) <= maxTargetDescLength {
		return desc
	}
	trimmed := desc[:maxTargetDescLength]
	for len(trimmed) > 0 && !utf8.ValidString(trimmed) {
		trimmed = trimmed[:len(trimmed)-1]
	}
	return trimmed
}

// SMSPhoneNumber extracts the phoneNumber from an sms contact-method config,
// or "" when missing or malformed.
func SMSPhoneNumber(config []byte) string {
	var parsed struct {
		PhoneNumber string `json:"phoneNumber"`
	}
	_ = json.Unmarshal(config, &parsed)
	return parsed.PhoneNumber
}

// smsDescFor labels the delivery row. GET /api/pages/:id returns these to
// every reader of the project, so the number is masked to its last 4 digits:
// the responder still recognises their own phone, nobody else learns it.
func smsDescFor(config []byte) string {
	number := SMSPhoneNumber(config)
	if number == "" {
		return "sms"
	}
	return notifications.MaskPhoneNumber(number) + " (sms)"
}

// ParseEmailOverride extracts the optional email override from an email
// contact-method config; "" means the account email is used.
func ParseEmailOverride(config []byte) string {
	var parsed struct {
		Email string `json:"email"`
	}
	_ = json.Unmarshal(config, &parsed)
	return parsed.Email
}

// EmailDeliveryFor builds the email adapter config (and display description)
// for a recipient, applying the override when present. Overrides are masked in
// the delivery log; the account email is not.
func EmailDeliveryFor(accountEmail string, override string) (json.RawMessage, string) {
	email := accountEmail
	desc := accountEmail
	if override != "" {
		email = override
		desc = maskEmail(override)
	}
	configJSON, _ := json.Marshal(map[string]any{"recipients": []string{email}})
	return configJSON, desc + " (email)"
}

// maskEmail keeps the first character of the local part and the whole domain.
func maskEmail(email string) string {
	at := strings.LastIndex(email, "@")
	if at <= 0 {
		return email
	}
	return email[:1] + "***" + email[at:]
}

// claimChannelDelivery records the delivery-log row for a plain-channel target
// (method_type "channel" for display) and enqueues the outbox delivery with
// the channel's real adapter type and a config snapshot. Channel messages
// carry the dashboard link, never an ack token: an anyone-can-click token in a
// shared room defeats attribution.
func claimChannelDelivery(tx *sql.Tx, page *models.Page, level int, channelId int, now time.Time) (int, error) {
	channel, err := transactional.NotificationChannelRepository.FindById(tx, channelId)
	if err != nil {
		return 0, err
	}
	if channel == nil || channel.ChannelType == "escalation" {
		traceway.CaptureException(fmt.Errorf("escalation target channel %d no longer exists or is not deliverable", channelId))
		return 0, nil
	}
	message := buildPageMessage(page, level, AckLink(page))
	scheduledFor := now
	notification := &models.PageNotification{
		PageId:       page.Id,
		Level:        level,
		Iteration:    page.RepeatIteration,
		TargetDesc:   "Channel: " + channel.Name,
		MethodType:   "channel",
		Status:       models.PageNotificationPending,
		ScheduledFor: &scheduledFor,
		CreatedAt:    now,
	}
	notificationId, err := transactional.PageNotificationRepository.Create(tx, notification)
	if err != nil {
		return 0, err
	}
	if _, err := outbox.Enqueue(tx, outbox.Delivery{
		Kind:               models.OutboxKindPage,
		AdapterType:        channel.ChannelType,
		AdapterConfig:      json.RawMessage(channel.Config),
		Message:            message,
		CancelKey:          outbox.PageCancelKey(page.Id),
		PageNotificationId: &notificationId,
	}); err != nil {
		return 0, err
	}
	return 1, nil
}

// buildPageMessage builds the delivery message. ackURL is per-delivery: user
// deliveries carry their own tokenized link, channel deliveries the dashboard
// link.
func buildPageMessage(page *models.Page, level int, ackURL string) notifications.Message {
	prefix := "[Page] "
	escalationLevel := 0
	if level > 0 {
		escalationLevel = level + 1
		prefix = fmt.Sprintf("[Page - escalation L%d] ", escalationLevel)
	}
	body := page.Body
	if body != "" {
		body += "\n\n"
	}
	body += "Acknowledge this page: " + ackURL

	severity := notifications.Severity(page.Severity)
	if severity == "" {
		severity = notifications.SeverityCritical
	}
	return notifications.Message{
		Subject:  prefix + page.Subject,
		Body:     body,
		Severity: severity,
		RuleType: page.RuleType,
		RuleName: page.RuleName,
		URL:      ackURL,
		Email: &models.NotificationEmail{
			Template: models.EmailTemplatePage,
			Page: &models.EmailPage{
				BodyText:        page.Body,
				RuleName:        page.RuleName,
				EventCount:      page.EventCount,
				EscalationLevel: escalationLevel,
			},
		},
	}
}

// newAckToken mints an opaque per-delivery acknowledge token. Only its SHA-256
// hash is stored; the plaintext exists solely inside the outgoing message.
func newAckToken() string {
	b := make([]byte, 32)
	if _, err := rand.Read(b); err != nil {
		panic(err)
	}
	return "twk_" + base64.RawURLEncoding.EncodeToString(b)
}

// appBaseURL is the dashboard origin for outgoing links. On-call deployments
// should set APP_BASE_URL; without it an ack link still has to be clickable.
func appBaseURL() string { return config.Config.PublicBaseURLOrDev() }

// ackURLFor builds the public no-login acknowledge link for a delivery token.
func ackURLFor(token string) string {
	return appBaseURL() + "/ack/" + token
}

// AckLink builds the absolute dashboard link that acknowledges a page. The
// project id is included so the dashboard can switch to the page's project —
// pages are resolved against the viewer's selected project.
func AckLink(page *models.Page) string {
	return appBaseURL() + "/on-call?page=" + strconv.Itoa(page.Id) + "&projectId=" + page.ProjectId.String()
}