Skip to content

The only tool you need to know what is happening and how to fix it.

17.3 KB
Raw Blame
//go:build !transactional_pg && !telemetry_ch && !telemetry_duckdb

package oncall

import (
	"context"
	"database/sql"
	"encoding/json"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/tracewayapp/traceway/backend/app/config"
	"github.com/tracewayapp/traceway/backend/app/db"
	"github.com/tracewayapp/traceway/backend/app/models"
	"github.com/tracewayapp/traceway/backend/app/outbox"
	"github.com/tracewayapp/traceway/backend/app/repositories/transactional"
	"github.com/tracewayapp/traceway/backend/app/repositories/transactional/shared"
)

func TestResolveUrgencyMatrix(t *testing.T) {
	cases := []struct {
		policy   string
		severity string
		want     string
	}{
		{"", "critical", models.UrgencyHigh},
		{"", "", models.UrgencyHigh},
		{"", "warning", models.UrgencyLow},
		{"", "info", models.UrgencyLow},
		{"auto", "critical", models.UrgencyHigh},
		{"auto", "info", models.UrgencyLow},
		{"high", "info", models.UrgencyHigh},
		{"low", "critical", models.UrgencyLow},
		{"junk", "critical", models.UrgencyHigh},
	}
	for _, tc := range cases {
		if got := ResolveUrgency(tc.policy, tc.severity); got != tc.want {
			t.Errorf("ResolveUrgency(%q, %q) = %q, want %q", tc.policy, tc.severity, got, tc.want)
		}
	}
}

func TestPolicyUrgencyValidationAndParsing(t *testing.T) {
	fixture := setupEscalatorDB(t)
	_, err := db.ExecuteTransaction(func(tx *sql.Tx) (struct{}, error) {
		valid := `{"steps":[{"targets":[{"type":"user","id":` + itoa(fixture.Alice) + `}],"delayMinutes":5}],"urgency":"high"}`
		if _, err := ValidatePolicyDefinition(tx, fixture.OrgId, []byte(valid)); err != nil {
			t.Errorf("high urgency should validate: %v", err)
		}
		bad := `{"steps":[{"targets":[{"type":"user","id":` + itoa(fixture.Alice) + `}],"delayMinutes":5}],"urgency":"shout"}`
		if _, err := ValidatePolicyDefinition(tx, fixture.OrgId, []byte(bad)); err == nil {
			t.Error("junk urgency should 422")
		}
		return struct{}{}, nil
	})
	if err != nil {
		t.Fatalf("tx: %v", err)
	}

	// Old definitions without the field parse to auto.
	definition, err := ParsePolicyDefinition([]byte(`{"schemaVersion":1,"steps":[],"repeatCount":0}`))
	if err != nil {
		t.Fatalf("parse: %v", err)
	}
	if definition.Urgency != "" {
		t.Errorf("legacy definition urgency = %q, want empty (auto)", definition.Urgency)
	}
}

func itoa(v int) string {
	return strconv.Itoa(v)
}

func createContactMethod(t *testing.T, userId int, methodType string, config string, enabled bool, verified bool) int {
	t.Helper()
	id, err := db.ExecuteTransaction(func(tx *sql.Tx) (int, error) {
		return transactional.UserContactMethodRepository.Create(tx, &models.UserContactMethod{
			UserId:     userId,
			MethodType: methodType,
			Config:     models.JSONText(config),
			Enabled:    enabled,
			Verified:   verified,
			CreatedAt:  time.Now().UTC(),
		})
	})
	if err != nil {
		t.Fatalf("create contact method: %v", err)
	}
	return id
}

func setRules(t *testing.T, userId int, urgency string, steps []models.UserNotificationRule) {
	t.Helper()
	_, err := db.ExecuteTransaction(func(tx *sql.Tx) (struct{}, error) {
		rules := make([]*models.UserNotificationRule, 0, len(steps))
		for i := range steps {
			rule := steps[i]
			rule.UserId = userId
			rule.Urgency = urgency
			rule.Position = i
			rule.CreatedAt = time.Now().UTC()
			rules = append(rules, &rule)
		}
		return struct{}{}, transactional.UserNotificationRuleRepository.ReplaceForUser(tx, userId, rules)
	})
	if err != nil {
		t.Fatalf("set rules: %v", err)
	}
}

func outboxRowsForPage(t *testing.T, pageId int) []*models.OutboxDelivery {
	t.Helper()
	rows, err := db.ExecuteTransaction(func(tx *sql.Tx) ([]*models.OutboxDelivery, error) {
		return transactional.OutboxRepository.FindCancellable(tx, outbox.PageCancelKey(pageId))
	})
	if err != nil {
		t.Fatalf("load outbox rows: %v", err)
	}
	return rows
}

func singleStepPolicy(userId int) string {
	return `{"schemaVersion":1,"steps":[{"targets":[{"type":"user","id":` + strconv.Itoa(userId) + `}],"delayMinutes":30}],"repeatCount":0}`
}

// enableTwilioForTest makes SMS a sendable method for the duration of one
// test; without credentials the escalator skips SMS methods entirely.
func enableTwilioForTest(t *testing.T) {
	t.Helper()
	previous := *config.Config
	config.Config.TwilioAccountSID = "ACtest"
	config.Config.TwilioAuthToken = "test-token"
	config.Config.TwilioFromNumber = "+15005550006"
	t.Cleanup(func() { *config.Config = previous })
}

func TestRuleChainEnqueuesStaggeredSteps(t *testing.T) {
	fixture := setupEscalatorDB(t)
	enableTwilioForTest(t)
	emailId := createContactMethod(t, fixture.Alice, "email", `{}`, true, true)
	smsId := createContactMethod(t, fixture.Alice, "sms", `{"phoneNumber":"+12025550123"}`, true, true)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{
		{ContactMethodId: smsId, DelayMinutes: 0},
		{ContactMethodId: emailId, DelayMinutes: 5},
	})

	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	page := openTestPageForPolicy(t, fixture, policyId, "chain1|/issues/abc")

	now := time.Now().UTC()
	runEscalatorTick(context.Background(), now)

	rows := pageNotifications(t, page.Id)
	if len(rows) != 2 {
		t.Fatalf("expected 2 chain rows, got %d", len(rows))
	}
	if rows[0].MethodType != "sms" || rows[0].ScheduledFor == nil {
		t.Errorf("first step = %+v, want immediate sms", rows[0])
	}
	if rows[1].MethodType != "email" || rows[1].ScheduledFor == nil {
		t.Fatalf("second step = %+v, want delayed email", rows[1])
	}
	if delay := rows[1].ScheduledFor.Sub(now); delay < 4*time.Minute || delay > 6*time.Minute {
		t.Errorf("email step scheduled in %v, want ~5m", delay)
	}
	if rows[0].AckTokenHash == "" || rows[1].AckTokenHash == "" || rows[0].AckTokenHash == rows[1].AckTokenHash {
		t.Error("each delivery row should carry a distinct ack token hash")
	}

	outboxRows := outboxRowsForPage(t, page.Id)
	if len(outboxRows) != 2 {
		t.Fatalf("expected 2 outbox rows, got %d", len(outboxRows))
	}
	var delayed *models.OutboxDelivery
	for _, row := range outboxRows {
		if row.AdapterType == "email" {
			delayed = row
		}
	}
	if delayed == nil {
		t.Fatal("expected an email outbox row")
	}
	if gap := delayed.NextAttemptAt.Sub(now); gap < 4*time.Minute || gap > 6*time.Minute {
		t.Errorf("email outbox NotBefore in %v, want ~5m", gap)
	}
}

func TestAckCancelsChainTail(t *testing.T) {
	fixture := setupEscalatorDB(t)
	emailId := createContactMethod(t, fixture.Alice, "email", `{}`, true, true)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{
		{ContactMethodId: emailId, DelayMinutes: 0},
		{ContactMethodId: emailId, DelayMinutes: 10},
	})

	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	page := openTestPageForPolicy(t, fixture, policyId, "chain2|/issues/abc")

	now := time.Now().UTC()
	tickAndDrain(t, now)

	rows := pageNotifications(t, page.Id)
	if len(rows) != 2 {
		t.Fatalf("expected 2 rows, got %d", len(rows))
	}
	if rows[0].Status != models.PageNotificationSent {
		t.Errorf("immediate step = %s, want sent", rows[0].Status)
	}
	if rows[1].Status != models.PageNotificationPending {
		t.Errorf("delayed step = %s, want pending", rows[1].Status)
	}

	acknowledged, err := db.ExecuteTransaction(func(tx *sql.Tx) (bool, error) {
		return AcknowledgePage(tx, page.Id, &fixture.Alice, AckViaDashboard, now)
	})
	if err != nil || !acknowledged {
		t.Fatalf("acknowledge: %v (%v)", err, acknowledged)
	}
	rows = pageNotifications(t, page.Id)
	if rows[1].Status != models.PageNotificationCancelled {
		t.Errorf("delayed step after ack = %s, want cancelled", rows[1].Status)
	}
	page = reloadPage(t, page.Id)
	if page.AcknowledgedVia != AckViaDashboard {
		t.Errorf("acknowledged_via = %q, want dashboard", page.AcknowledgedVia)
	}
}

func TestRuleChainSkipsUnusableStepsAndFallsBack(t *testing.T) {
	fixture := setupEscalatorDB(t)
	disabledId := createContactMethod(t, fixture.Alice, "email", `{}`, false, true)
	unverifiedSmsId := createContactMethod(t, fixture.Alice, "sms", `{"phoneNumber":"+12025550123"}`, true, false)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{
		{ContactMethodId: disabledId, DelayMinutes: 0},
		{ContactMethodId: unverifiedSmsId, DelayMinutes: 5},
	})

	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	page := openTestPageForPolicy(t, fixture, policyId, "chain3|/issues/abc")
	runEscalatorTick(context.Background(), time.Now().UTC())

	rows := pageNotifications(t, page.Id)
	if len(rows) != 1 {
		t.Fatalf("expected 1 fallback row, got %d", len(rows))
	}
	if rows[0].MethodType != "email" || rows[0].TargetDesc != "[email protected] (email)" {
		t.Errorf("fallback should page the account email, got %+v", rows[0])
	}
}

// Dropping the leading zero-delay step must not postpone the first page.
func TestRuleChainRebasesWhenTheLeadingStepIsDropped(t *testing.T) {
	fixture := setupEscalatorDB(t)
	disabledId := createContactMethod(t, fixture.Alice, "email", `{"email":"[email protected]"}`, false, true)
	secondId := createContactMethod(t, fixture.Alice, "email", `{"email":"[email protected]"}`, true, true)
	thirdId := createContactMethod(t, fixture.Alice, "email", `{"email":"[email protected]"}`, true, true)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{
		{ContactMethodId: disabledId, DelayMinutes: 0},
		{ContactMethodId: secondId, DelayMinutes: 15},
		{ContactMethodId: thirdId, DelayMinutes: 20},
	})

	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	page := openTestPageForPolicy(t, fixture, policyId, "rebase1|/issues/abc")
	now := time.Now().UTC()
	runEscalatorTick(context.Background(), now)

	rows := pageNotifications(t, page.Id)
	if len(rows) != 2 {
		t.Fatalf("expected the 2 surviving steps, got %d", len(rows))
	}
	if rows[0].TargetDesc != "s***@example.com (email)" {
		t.Errorf("first surviving step = %q, want no delay suffix", rows[0].TargetDesc)
	}
	if rows[0].ScheduledFor == nil || rows[0].ScheduledFor.After(now.Add(time.Minute)) {
		t.Errorf("first surviving step scheduled for %v, want immediately", rows[0].ScheduledFor)
	}
	// 20m - 15m: the spacing between the surviving steps is preserved.
	if rows[1].TargetDesc != "t***@example.com (email), +5m" {
		t.Errorf("second surviving step = %q, want a 5 minute gap", rows[1].TargetDesc)
	}
}

// An intact leading delay is kept; the rebase must not erase it.
func TestRuleChainKeepsAnIntentionalLeadingDelay(t *testing.T) {
	fixture := setupEscalatorDB(t)
	firstId := createContactMethod(t, fixture.Alice, "email", `{"email":"[email protected]"}`, true, true)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{
		{ContactMethodId: firstId, DelayMinutes: 10},
	})

	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	page := openTestPageForPolicy(t, fixture, policyId, "rebase2|/issues/abc")
	runEscalatorTick(context.Background(), time.Now().UTC())

	rows := pageNotifications(t, page.Id)
	if len(rows) != 1 || rows[0].TargetDesc != "f***@example.com (email), +10m" {
		t.Errorf("intact leading delay should survive, got %+v", rows)
	}
}

// An oversized target_desc must be clamped, not fail the claim insert.
func TestOversizedTargetDescIsClamped(t *testing.T) {
	fixture := setupEscalatorDB(t)
	long := strings.Repeat("a", 400) + "@example.com"
	methodId := createContactMethod(t, fixture.Alice, "email", `{"email":"`+long+`"}`, true, true)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{
		{ContactMethodId: methodId, DelayMinutes: 0},
	})

	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	page := openTestPageForPolicy(t, fixture, policyId, "clamp1|/issues/abc")
	runEscalatorTick(context.Background(), time.Now().UTC())

	rows := pageNotifications(t, page.Id)
	if len(rows) != 1 {
		t.Fatalf("expected 1 notification, got %d", len(rows))
	}
	if len(rows[0].TargetDesc) > maxTargetDescLength {
		t.Errorf("target desc is %d chars, want at most %d", len(rows[0].TargetDesc), maxTargetDescLength)
	}
}

// An SMS method can outlive its transport: the credentials are removed after
// the method was verified. The page must not vanish into an unsendable
// channel, so SMS is skipped and the account-email fallback still fires.
func TestSmsMethodsSkippedWithoutTwilio(t *testing.T) {
	fixture := setupEscalatorDB(t)
	if config.Config.TwilioEnabled() {
		t.Skip("Twilio configured in this environment")
	}
	smsId := createContactMethod(t, fixture.Alice, "sms", `{"phoneNumber":"+12025550123"}`, true, true)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{{ContactMethodId: smsId, DelayMinutes: 0}})

	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	page := openTestPageForPolicy(t, fixture, policyId, "nosms|/issues/abc")
	runEscalatorTick(context.Background(), time.Now().UTC())

	rows := pageNotifications(t, page.Id)
	if len(rows) != 1 {
		t.Fatalf("expected 1 fallback row, got %d", len(rows))
	}
	if rows[0].MethodType != "email" || rows[0].TargetDesc != "[email protected] (email)" {
		t.Errorf("unsendable sms should fall back to the account email, got %+v", rows[0])
	}
}

func TestLowUrgencyPageRunsLowChain(t *testing.T) {
	fixture := setupEscalatorDB(t)
	emailId := createContactMethod(t, fixture.Alice, "email", `{"email":"[email protected]"}`, true, true)
	smsId := createContactMethod(t, fixture.Alice, "sms", `{"phoneNumber":"+12025550123"}`, true, true)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{{ContactMethodId: smsId, DelayMinutes: 0}})
	// setRules replaces the WHOLE rule set, so write both chains at once.
	_, err := db.ExecuteTransaction(func(tx *sql.Tx) (struct{}, error) {
		now := time.Now().UTC()
		return struct{}{}, transactional.UserNotificationRuleRepository.ReplaceForUser(tx, fixture.Alice, []*models.UserNotificationRule{
			{UserId: fixture.Alice, Urgency: models.UrgencyHigh, Position: 0, DelayMinutes: 0, ContactMethodId: smsId, CreatedAt: now},
			{UserId: fixture.Alice, Urgency: models.UrgencyLow, Position: 0, DelayMinutes: 0, ContactMethodId: emailId, CreatedAt: now},
		})
	})
	if err != nil {
		t.Fatalf("set chains: %v", err)
	}

	// warning severity + auto urgency -> low chain (email), not sms.
	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	if _, err := openPage(openPageParams{
		PolicyId: policyId, ProjectId: fixture.ProjectId, RuleName: "r", RuleType: "new_error",
		Subject: "warn", Severity: "warning", DedupKey: "chain4|/issues/w",
	}); err != nil {
		t.Fatalf("open page: %v", err)
	}
	page := findPageByDedupKey(t, "chain4|/issues/w")
	if page.Urgency != models.UrgencyLow {
		t.Fatalf("page urgency = %q, want low", page.Urgency)
	}
	runEscalatorTick(context.Background(), time.Now().UTC())

	rows := pageNotifications(t, page.Id)
	if len(rows) != 1 || rows[0].MethodType != "email" || rows[0].TargetDesc != "l***@example.com (email)" {
		t.Errorf("low chain should run the email step, got %+v", rows)
	}
}

func TestAckTokenFlow(t *testing.T) {
	fixture := setupEscalatorDB(t)
	emailId := createContactMethod(t, fixture.Alice, "email", `{}`, true, true)
	setRules(t, fixture.Alice, models.UrgencyHigh, []models.UserNotificationRule{{ContactMethodId: emailId, DelayMinutes: 0}})

	policyId := createPolicy(t, fixture.OrgId, singleStepPolicy(fixture.Alice))
	page := openTestPageForPolicy(t, fixture, policyId, "token1|/issues/abc")
	runEscalatorTick(context.Background(), time.Now().UTC())

	// The plaintext token exists only inside the outgoing message.
	outboxRows := outboxRowsForPage(t, page.Id)
	if len(outboxRows) != 1 {
		t.Fatalf("expected 1 outbox row, got %d", len(outboxRows))
	}
	var msg models.NotificationMessage
	if err := jsonUnmarshalForTest(outboxRows[0].Message, &msg); err != nil {
		t.Fatalf("decode message: %v", err)
	}
	idx := strings.LastIndex(msg.URL, "/ack/")
	if idx < 0 {
		t.Fatalf("message URL carries no ack link: %q", msg.URL)
	}
	token := msg.URL[idx+len("/ack/"):]
	if !strings.HasPrefix(token, "twk_") {
		t.Fatalf("unexpected token shape %q", token)
	}

	// Token resolves to the delivery row, attributed to Alice.
	notification, err := db.ExecuteTransaction(func(tx *sql.Tx) (*models.PageNotification, error) {
		return transactional.PageNotificationRepository.FindByAckTokenHash(tx, shared.HashAuthToken(token))
	})
	if err != nil || notification == nil {
		t.Fatalf("token lookup failed: %v (%v)", err, notification)
	}
	if notification.UserId == nil || *notification.UserId != fixture.Alice {
		t.Fatalf("token attributed to %v, want alice", notification.UserId)
	}

	// Link-ack records via + attribution and stops the escalation.
	acknowledged, err := db.ExecuteTransaction(func(tx *sql.Tx) (bool, error) {
		return AcknowledgePage(tx, page.Id, notification.UserId, AckViaLink, time.Now().UTC())
	})
	if err != nil || !acknowledged {
		t.Fatalf("link ack failed: %v (%v)", err, acknowledged)
	}
	page = reloadPage(t, page.Id)
	if page.AcknowledgedVia != AckViaLink || page.AcknowledgedBy == nil || *page.AcknowledgedBy != fixture.Alice {
		t.Errorf("link ack not recorded: via=%q by=%v", page.AcknowledgedVia, page.AcknowledgedBy)
	}

	// Wrong token finds nothing.
	missing, err := db.ExecuteTransaction(func(tx *sql.Tx) (*models.PageNotification, error) {
		return transactional.PageNotificationRepository.FindByAckTokenHash(tx, shared.HashAuthToken("twk_bogus"))
	})
	if err != nil || missing != nil {
		t.Errorf("bogus token should resolve to nothing, got %v (%v)", missing, err)
	}
}

func jsonUnmarshalForTest(data []byte, v any) error {
	return json.Unmarshal(data, v)
}