Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(rollout): emit PIPELINE_COMPLETED after BatchSkipTasks resolves a failure (BYT-9398) (#20212)

zchpeter35c30da829d2
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.go
index 6550f24887..ba63b0e576 100644
--- a/backend/api/v1/rollout_service.go
+++ b/backend/api/v1/rollout_service.go
@@ -741,12 +741,6 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("rollout (plan) %v not found", planID))
}
- // Reset notification state so user gets fresh feedback on retry
- if err := s.store.ResetPlanWebhookDelivery(ctx, projectID, planID); err != nil {
- slog.Error("failed to reset plan webhook delivery", log.BBError(err))
- // Don't fail the request - notification is non-critical
- }
-
issueN, err := s.store.GetIssue(ctx, &store.FindIssueMessage{
Workspace: common.GetWorkspaceIDFromContext(ctx),
ProjectIDs: []string{projectID},
@@ -844,6 +838,21 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request
}
})
+ // Reset notification state so the user gets fresh feedback on this retry.
+ // Placed AFTER all validation (auth, approval, task parse) so a rejected
+ // request leaves the dedup ledger untouched, but BEFORE
+ // CreatePendingTaskRuns so the pending-task scheduler — which polls
+ // independently on its own ticker — cannot observe the new task_run rows
+ // before the stale plan_webhook_delivery row is cleared. Without this
+ // ordering, a fast-failing retry could be picked up, fail, and try to
+ // claim PIPELINE_FAILED against the stale row (ON CONFLICT DO NOTHING),
+ // and the API's later reset would clear the row with no pending event
+ // left to re-fire. Errors are logged and swallowed so a transient DB
+ // hiccup doesn't fail the user-facing run request.
+ if err := s.store.ResetPlanWebhookDelivery(ctx, projectID, planID); err != nil {
+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))
+ }
+
if err := s.store.CreatePendingTaskRuns(ctx, user.Email, taskRunCreates...); err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create pending task runs, error %v", err))
}
@@ -857,6 +866,14 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request
// BatchSkipTasks skips tasks in batch.
func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Request[v1pb.BatchSkipTasksRequest]) (*connect.Response[v1pb.BatchSkipTasksResponse], error) {
request := req.Msg
+ // Reject empty input early. Without this guard the request would skip the
+ // task-parse loop, the environmentSet-based permission check, and arrive
+ // at a no-op store.BatchSkipTasks — but still run ResetPlanWebhookDelivery
+ // and signal the completion check, mutating dedup state for a request that
+ // did nothing. Mirrors the BatchRunTasks pattern.
+ if len(request.Tasks) == 0 {
+ return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("The tasks in request cannot be empty"))
+ }
projectID, planID, _, err := common.GetProjectIDPlanIDMaybeStageID(request.Parent)
if err != nil {
return nil, connect.NewError(connect.CodeInvalidArgument, err)
@@ -936,6 +953,17 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Reques
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to skip tasks"))
}
+ // Reset notification state so PIPELINE_COMPLETED can fire after skipping a
+ // failed task (the BYT-9398 fix). Placed after authorization + the
+ // successful BatchSkipTasks so a rejected request (auth fail, unknown
+ // task, etc.) leaves the dedup ledger untouched. Must run before the
+ // completion-check signal below so the async claim sees the cleared row.
+ // Errors are logged and swallowed so a DB hiccup doesn't fail the
+ // user-facing skip request.
+ if err := s.store.ResetPlanWebhookDelivery(ctx, projectID, planID); err != nil {
+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))
+ }
+
// Signal to check if plan is complete and successful (may send PIPELINE_COMPLETED)
s.bus.PlanCompletionCheckChan <- bus.PlanRef{ProjectID: projectID, PlanID: planID}
diff --git a/backend/tests/webhook_helpers_test.go b/backend/tests/webhook_helpers_test.go
new file mode 100644
index 0000000000..75ebe744b6
--- /dev/null
+++ b/backend/tests/webhook_helpers_test.go
@@ -0,0 +1,705 @@
+package tests
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "fmt"
+ "path"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "connectrpc.com/connect"
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/genproto/googleapis/type/expr"
+ "google.golang.org/protobuf/types/known/fieldmaskpb"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// firstSlackSectionText returns the text of the first Slack section block in the
+// captured payload, or "" if none. attachments[0].blocks[0].text.text holds the
+// event title decorated with emoji and Slack <link|title> markup — see
+// backend/plugin/webhook/slack/slack_test.go:42-47, 85.
+func firstSlackSectionText(body []byte) string {
+ var payload map[string]any
+ if err := json.Unmarshal(body, &payload); err != nil {
+ return ""
+ }
+ attachments, ok := payload["attachments"].([]any)
+ if !ok || len(attachments) == 0 {
+ return ""
+ }
+ att, ok := attachments[0].(map[string]any)
+ if !ok {
+ return ""
+ }
+ blocks, ok := att["blocks"].([]any)
+ if !ok || len(blocks) == 0 {
+ return ""
+ }
+ block0, ok := blocks[0].(map[string]any)
+ if !ok || block0["type"] != "section" {
+ return ""
+ }
+ textMap, ok := block0["text"].(map[string]any)
+ if !ok {
+ return ""
+ }
+ text, ok := textMap["text"].(string)
+ if !ok {
+ return ""
+ }
+ return text
+}
+
+// matchesEvent reports whether a captured Slack payload matches the given
+// project and event title. Project filtering prevents cross-subtest
+// contamination from the async webhook dispatcher.
+func matchesEvent(req webhookRequest, projectName, eventTitle string) bool {
+ body := string(req.Body)
+ if !strings.Contains(body, projectName) {
+ return false
+ }
+ return strings.Contains(firstSlackSectionText(req.Body), eventTitle)
+}
+
+// webhookWaitTimeout is the deadline for waitForWebhookCount and the issue-state
+// wait helpers. Webhook delivery + completion-check fanout takes <5s in practice;
+// 30s gives ample headroom under CI load.
+const webhookWaitTimeout = 30 * time.Second
+
+// waitForWebhookCount blocks until at least n webhooks for (project, eventTitle)
+// have arrived, or fails the test after webhookWaitTimeout.
+func waitForWebhookCount(t *testing.T, c *webhookCollector, projectName, eventTitle string, n int) {
+ t.Helper()
+ deadline := time.Now().Add(webhookWaitTimeout)
+ for {
+ count := countWebhooksFor(c, projectName, eventTitle)
+ if count >= n {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("timed out waiting for %d %q webhooks on %s; got %d after %s",
+ n, eventTitle, projectName, count, webhookWaitTimeout)
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+}
+
+// requireWebhookCount asserts the exact count of webhooks for (project, eventTitle).
+func requireWebhookCount(t *testing.T, c *webhookCollector, projectName, eventTitle string, n int) {
+ t.Helper()
+ got := countWebhooksFor(c, projectName, eventTitle)
+ require.Equalf(t, n, got, "expected %d %q webhooks on %s, got %d", n, eventTitle, projectName, got)
+}
+
+func countWebhooksFor(c *webhookCollector, projectName, eventTitle string) int {
+ n := 0
+ for _, req := range c.getRequests() {
+ if matchesEvent(req, projectName, eventTitle) {
+ n++
+ }
+ }
+ return n
+}
+
+// createTestProject creates a fresh project (default AllowSelfApproval=true;
+// approval-flow subtests call disableSelfApproval).
+func (ctl *controller) createTestProject(ctx context.Context, t *testing.T, prefix string) *v1pb.Project {
+ t.Helper()
+ pid := generateRandomString(prefix)
+ resp, err := ctl.projectServiceClient.CreateProject(ctx, connect.NewRequest(&v1pb.CreateProjectRequest{
+ ProjectId: pid,
+ Project: &v1pb.Project{
+ Name: fmt.Sprintf("projects/%s", pid),
+ Title: prefix,
+ AllowSelfApproval: true,
+ },
+ }))
+ require.NoError(t, err)
+ return resp.Msg
+}
+
+// addWebhookForEvents adds a Slack webhook to the project subscribed to the
+// given event types only.
+func addWebhookForEvents(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, url string, events []v1pb.Activity_Type) {
+ t.Helper()
+ _, err := ctl.projectServiceClient.AddWebhook(ctx, connect.NewRequest(&v1pb.AddWebhookRequest{
+ Project: project.Name,
+ Webhook: &v1pb.Webhook{
+ Type: v1pb.WebhookType_SLACK,
+ Title: "test-webhook-" + generateRandomString("hook"),
+ Url: url,
+ NotificationTypes: events,
+ },
+ }))
+ require.NoError(t, err)
+}
+
+// dbTargetName returns the canonical instance/databases/<name> resource form.
+func dbTargetName(instance *v1pb.Instance, dbName string) string {
+ return fmt.Sprintf("%s/databases/%s", instance.Name, dbName)
+}
+
+// seedPassingSheet creates a sheet whose SQL trivially succeeds.
+func seedPassingSheet(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project) string {
+ t.Helper()
+ resp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{
+ Parent: project.Name,
+ Sheet: &v1pb.Sheet{Content: []byte("SELECT 1;")},
+ }))
+ require.NoError(t, err)
+ return resp.Msg.Name
+}
+
+// seedFailingSheet creates a sheet whose SQL fails until unblockFailingTask runs.
+func seedFailingSheet(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project) string {
+ t.Helper()
+ resp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{
+ Parent: project.Name,
+ Sheet: &v1pb.Sheet{Content: []byte("INSERT INTO __force_fail_target VALUES(1);")},
+ }))
+ require.NoError(t, err)
+ return resp.Msg.Name
+}
+
+// unblockFailingTask creates the missing table inside the SQLite database file
+// so subsequent runs of seedFailingSheet's SQL succeed. db.Close() flushes
+// WAL/journal so the file is consistent before the retry is enqueued.
+func unblockFailingTask(t *testing.T, instanceDir, dbName string) {
+ t.Helper()
+ dbPath := filepath.Join(instanceDir, dbName+".db")
+ db, err := sql.Open("sqlite3", dbPath)
+ require.NoError(t, err)
+ defer func() {
+ if cerr := db.Close(); cerr != nil {
+ t.Errorf("close sqlite handle for %s: %v", dbPath, cerr)
+ }
+ }()
+ _, err = db.Exec("CREATE TABLE IF NOT EXISTS __force_fail_target(id INT);")
+ require.NoError(t, err)
+}
+
+type taskSpec struct {
+ sheetName string
+ dbTarget string // full instance/databases/<name>
+}
+
+// createPlanWithSpecs creates a plan with one ChangeDatabaseConfig spec per entry.
+func createPlanWithSpecs(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, specs []taskSpec) *v1pb.Plan {
+ t.Helper()
+ var planSpecs []*v1pb.Plan_Spec
+ for _, s := range specs {
+ planSpecs = append(planSpecs, &v1pb.Plan_Spec{
+ Id: uuid.NewString(),
+ Config: &v1pb.Plan_Spec_ChangeDatabaseConfig{
+ ChangeDatabaseConfig: &v1pb.Plan_ChangeDatabaseConfig{
+ Targets: []string{s.dbTarget},
+ Sheet: s.sheetName,
+ },
+ },
+ })
+ }
+ resp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{
+ Parent: project.Name,
+ Plan: &v1pb.Plan{Specs: planSpecs},
+ }))
+ require.NoError(t, err)
+ return resp.Msg
+}
+
+func createRolloutOnly(ctx context.Context, t *testing.T, ctl *controller, plan *v1pb.Plan) *v1pb.Rollout {
+ t.Helper()
+ resp, err := ctl.rolloutServiceClient.CreateRollout(ctx, connect.NewRequest(&v1pb.CreateRolloutRequest{Parent: plan.Name}))
+ require.NoError(t, err)
+ return resp.Msg
+}
+
+// runAllTasks materializes the rollout and starts every task in every stage.
+func runAllTasks(ctx context.Context, t *testing.T, ctl *controller, plan *v1pb.Plan) *v1pb.Rollout {
+ t.Helper()
+ rollout := createRolloutOnly(ctx, t, ctl, plan)
+ for _, stage := range rollout.Stages {
+ var taskNames []string
+ for _, task := range stage.Tasks {
+ taskNames = append(taskNames, task.Name)
+ }
+ if len(taskNames) == 0 {
+ continue
+ }
+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{
+ Parent: stage.Name,
+ Tasks: taskNames,
+ }))
+ require.NoError(t, err)
+ }
+ return rollout
+}
+
+func refreshRollout(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) *v1pb.Rollout {
+ t.Helper()
+ resp, err := ctl.rolloutServiceClient.GetRollout(ctx, connect.NewRequest(&v1pb.GetRolloutRequest{Name: rollout.Name}))
+ require.NoError(t, err)
+ return resp.Msg
+}
+
+// findTaskByDB scans the rollout for a task targeting dbResource.
+func findTaskByDB(t *testing.T, rollout *v1pb.Rollout, dbResource string) (stageName, taskName string) {
+ t.Helper()
+ for _, stage := range rollout.Stages {
+ for _, task := range stage.Tasks {
+ if task.Target == dbResource {
+ return stage.Name, task.Name
+ }
+ }
+ }
+ t.Fatalf("no task targets database %s in rollout %s", dbResource, rollout.Name)
+ return "", ""
+}
+
+func runTaskByDB(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout, dbResource string) {
+ t.Helper()
+ stageName, taskName := findTaskByDB(t, rollout, dbResource)
+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{
+ Parent: stageName,
+ Tasks: []string{taskName},
+ }))
+ require.NoError(t, err)
+}
+
+func skipTaskByDB(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout, dbResource string) {
+ t.Helper()
+ stageName, taskName := findTaskByDB(t, rollout, dbResource)
+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{
+ Parent: stageName,
+ Tasks: []string{taskName},
+ Reason: "test: skip task by db",
+ }))
+ require.NoError(t, err)
+}
+
+// skipFailedTasks finds every FAILED task in the rollout and skips them.
+func skipFailedTasks(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) {
+ t.Helper()
+ fresh := refreshRollout(ctx, t, ctl, rollout)
+ perStage := map[string][]string{}
+ for _, stage := range fresh.Stages {
+ for _, task := range stage.Tasks {
+ if task.Status == v1pb.Task_FAILED {
+ perStage[stage.Name] = append(perStage[stage.Name], task.Name)
+ }
+ }
+ }
+ require.NotEmpty(t, perStage, "expected at least one failed task to skip")
+ for stageName, names := range perStage {
+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{
+ Parent: stageName,
+ Tasks: names,
+ Reason: "test: skip failed tasks",
+ }))
+ require.NoError(t, err)
+ }
+}
+
+func skipAllTasks(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) {
+ t.Helper()
+ fresh := refreshRollout(ctx, t, ctl, rollout)
+ for _, stage := range fresh.Stages {
+ var names []string
+ for _, task := range stage.Tasks {
+ names = append(names, task.Name)
+ }
+ if len(names) == 0 {
+ continue
+ }
+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{
+ Parent: stage.Name,
+ Tasks: names,
+ Reason: "test: skip all",
+ }))
+ require.NoError(t, err)
+ }
+}
+
+// retryFailedTasks reruns BatchRunTasks on every FAILED task. Caller decides
+// whether to call unblockFailingTask first (to make the retry pass) or not
+// (to test that BatchRunTasks resets the dedup row so a second
+// PIPELINE_FAILED can fire).
+func retryFailedTasks(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) {
+ t.Helper()
+ fresh := refreshRollout(ctx, t, ctl, rollout)
+ perStage := map[string][]string{}
+ for _, stage := range fresh.Stages {
+ for _, task := range stage.Tasks {
+ if task.Status == v1pb.Task_FAILED {
+ perStage[stage.Name] = append(perStage[stage.Name], task.Name)
+ }
+ }
+ }
+ require.NotEmpty(t, perStage)
+ for stageName, names := range perStage {
+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{
+ Parent: stageName,
+ Tasks: names,
+ }))
+ require.NoError(t, err)
+ }
+}
+
+// waitForTaskStatus polls the rollout until the task targeting dbResource has
+// the requested status, or fails the test.
+func waitForTaskStatus(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout, dbResource string, want v1pb.Task_Status, timeout time.Duration) {
+ t.Helper()
+ deadline := time.Now().Add(timeout)
+ for {
+ fresh := refreshRollout(ctx, t, ctl, rollout)
+ for _, stage := range fresh.Stages {
+ for _, task := range stage.Tasks {
+ if task.Target != dbResource {
+ continue
+ }
+ if task.Status == want {
+ return
+ }
+ }
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("timed out waiting for task on %s to reach %s", dbResource, want)
+ }
+ time.Sleep(200 * time.Millisecond)
+ }
+}
+
+// waitForAllTasksTerminal polls until every task in the rollout is in a
+// terminal status (DONE, FAILED, SKIPPED, or CANCELED). Used before
+// "skip remaining" actions to avoid races with still-running tasks. On
+// timeout, dumps per-task status to t.Logf so failures are debuggable
+// without re-running.
+func waitForAllTasksTerminal(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout, timeout time.Duration) {
+ t.Helper()
+ deadline := time.Now().Add(timeout)
+ isTerminal := func(s v1pb.Task_Status) bool {
+ return s == v1pb.Task_DONE || s == v1pb.Task_FAILED || s == v1pb.Task_SKIPPED || s == v1pb.Task_CANCELED
+ }
+ var fresh *v1pb.Rollout
+ for {
+ fresh = refreshRollout(ctx, t, ctl, rollout)
+ allTerminal := true
+ for _, stage := range fresh.Stages {
+ for _, task := range stage.Tasks {
+ if !isTerminal(task.Status) {
+ allTerminal = false
+ }
+ }
+ }
+ if allTerminal {
+ return
+ }
+ if time.Now().After(deadline) {
+ for _, stage := range fresh.Stages {
+ for _, task := range stage.Tasks {
+ t.Logf("non-terminal task: %s status=%s target=%s", task.Name, task.Status, task.Target)
+ }
+ }
+ t.Fatalf("timed out waiting for all tasks terminal on rollout %s after %s", rollout.Name, timeout)
+ }
+ time.Sleep(200 * time.Millisecond)
+ }
+}
+
+// installWorkspaceApprovalRule writes a WORKSPACE_APPROVAL setting rule that
+// scopes to the given project via CEL and registers a cleanup that restores
+// the prior workspace setting. flowRoles is the ordered list of roles for
+// sequential approval steps (one role per step).
+func installWorkspaceApprovalRule(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, flowRoles []string) {
+ t.Helper()
+ projectID := path.Base(project.Name)
+
+ prior := snapshotWorkspaceApproval(ctx, t, ctl)
+
+ rules := []*v1pb.WorkspaceApprovalSetting_Rule{{
+ Source: v1pb.WorkspaceApprovalSetting_Rule_SOURCE_UNSPECIFIED,
+ Condition: &expr.Expr{
+ Expression: fmt.Sprintf(`resource.project_id == "%s"`, projectID),
+ },
+ Template: &v1pb.ApprovalTemplate{
+ Title: "Test approval flow for " + projectID,
+ Flow: &v1pb.ApprovalFlow{Roles: flowRoles},
+ },
+ }}
+ if prior != nil {
+ // Append to the snapshotted rules. Each subtest's t.Cleanup restores
+ // its own snapshot, so LIFO ordering of cleanups still ends with the
+ // truly-original workspace state. If a future test installs twice in
+ // the same subtest, cleanups run LIFO and each writes the snapshot
+ // taken at its install time, so the final state matches the original.
+ rules = append(prior.Rules, rules...)
+ }
+
+ writeWorkspaceApproval(ctx, t, ctl, &v1pb.WorkspaceApprovalSetting{Rules: rules})
+ t.Cleanup(func() {
+ restoreWorkspaceApproval(t, ctl, prior)
+ })
+}
+
+// clearWorkspaceApprovalRules removes all workspace approval rules (including the
+// default catch-all) and restores the prior setting on cleanup. Use this in
+// subtests that assert ISSUE_APPROVAL_REQUESTED does NOT fire when truly no rule
+// applies, or when a custom multi-step flow must not coexist with the default
+// projectOwner-only rule.
+func clearWorkspaceApprovalRules(ctx context.Context, t *testing.T, ctl *controller) {
+ t.Helper()
+
+ prior := snapshotWorkspaceApproval(ctx, t, ctl)
+ writeWorkspaceApproval(ctx, t, ctl, &v1pb.WorkspaceApprovalSetting{})
+ t.Cleanup(func() {
+ restoreWorkspaceApproval(t, ctl, prior)
+ })
+}
+
+// snapshotWorkspaceApproval reads the current WORKSPACE_APPROVAL setting and
+// fails the test if the read errors — silent failures here would let a later
+// "restore" wipe the workspace's real config.
+func snapshotWorkspaceApproval(ctx context.Context, t *testing.T, ctl *controller) *v1pb.WorkspaceApprovalSetting {
+ t.Helper()
+ resp, err := ctl.settingServiceClient.GetSetting(ctx, connect.NewRequest(&v1pb.GetSettingRequest{
+ Name: "settings/WORKSPACE_APPROVAL",
+ }))
+ require.NoError(t, err, "snapshot WORKSPACE_APPROVAL setting")
+ return resp.Msg.Value.GetWorkspaceApproval() // may be nil when no rules are set
+}
+
+func writeWorkspaceApproval(ctx context.Context, t *testing.T, ctl *controller, value *v1pb.WorkspaceApprovalSetting) {
+ t.Helper()
+ _, err := ctl.settingServiceClient.UpdateSetting(ctx, connect.NewRequest(&v1pb.UpdateSettingRequest{
+ AllowMissing: true,
+ Setting: &v1pb.Setting{
+ Name: "settings/WORKSPACE_APPROVAL",
+ Value: &v1pb.SettingValue{
+ Value: &v1pb.SettingValue_WorkspaceApproval{WorkspaceApproval: value},
+ },
+ },
+ }))
+ require.NoError(t, err, "write WORKSPACE_APPROVAL setting")
+}
+
+// restoreWorkspaceApproval writes the snapshotted setting back. Runs from
+// t.Cleanup; logs failures loudly so a corrupted teardown doesn't bleed into
+// later subtests as an order-dependent flake.
+func restoreWorkspaceApproval(t *testing.T, ctl *controller, prior *v1pb.WorkspaceApprovalSetting) {
+ t.Helper()
+ restored := prior
+ if restored == nil {
+ restored = &v1pb.WorkspaceApprovalSetting{}
+ }
+ if _, err := ctl.settingServiceClient.UpdateSetting(context.Background(), connect.NewRequest(&v1pb.UpdateSettingRequest{
+ AllowMissing: true,
+ Setting: &v1pb.Setting{
+ Name: "settings/WORKSPACE_APPROVAL",
+ Value: &v1pb.SettingValue{
+ Value: &v1pb.SettingValue_WorkspaceApproval{WorkspaceApproval: restored},
+ },
+ },
+ })); err != nil {
+ t.Errorf("failed to restore WORKSPACE_APPROVAL setting in cleanup: %v", err)
+ }
+}
+
+func disableSelfApproval(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project) {
+ t.Helper()
+ _, err := ctl.projectServiceClient.UpdateProject(ctx, connect.NewRequest(&v1pb.UpdateProjectRequest{
+ Project: &v1pb.Project{
+ Name: project.Name,
+ AllowSelfApproval: false,
+ },
+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"allow_self_approval"}},
+ }))
+ require.NoError(t, err)
+}
+
+type testApprover struct {
+ Email string
+ Password string
+}
+
+func provisionApprover(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, suffix, projectRole string) testApprover {
+ t.Helper()
+ email := fmt.Sprintf("approver-%[email protected]", suffix)
+ password := "1024bytebase"
+
+ newUser, err := ctl.userServiceClient.CreateUser(ctx, connect.NewRequest(&v1pb.CreateUserRequest{
+ User: &v1pb.User{Email: email, Password: password, Title: "Approver " + suffix},
+ }))
+ require.NoError(t, err)
+
+ _, err = ctl.addMemberToWorkspaceIAM(ctx, newUser.Msg.Workspace, fmt.Sprintf("user:%s", email), "roles/workspaceMember")
+ require.NoError(t, err)
+
+ policyResp, err := ctl.projectServiceClient.GetIamPolicy(ctx, connect.NewRequest(&v1pb.GetIamPolicyRequest{
+ Resource: project.Name,
+ }))
+ require.NoError(t, err)
+ policy := policyResp.Msg
+ policy.Bindings = append(policy.Bindings, &v1pb.Binding{
+ Role: projectRole,
+ Members: []string{fmt.Sprintf("user:%s", email)},
+ })
+ _, err = ctl.projectServiceClient.SetIamPolicy(ctx, connect.NewRequest(&v1pb.SetIamPolicyRequest{
+ Resource: project.Name,
+ Policy: policy,
+ }))
+ require.NoError(t, err)
+
+ return testApprover{Email: email, Password: password}
+}
+
+// withImpersonation logs in as the given approver, runs fn with that identity,
+// and restores the original token via defer.
+func withImpersonation(ctx context.Context, t *testing.T, ctl *controller, who testApprover, fn func()) {
+ t.Helper()
+ loginResp, err := ctl.authServiceClient.Login(ctx, connect.NewRequest(&v1pb.LoginRequest{
+ Email: who.Email,
+ Password: who.Password,
+ }))
+ require.NoError(t, err)
+
+ original := ctl.authInterceptor.token
+ ctl.authInterceptor.token = loginResp.Msg.Token
+ defer func() { ctl.authInterceptor.token = original }()
+
+ fn()
+}
+
+func createIssueForPlan(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, plan *v1pb.Plan, title string) *v1pb.Issue {
+ t.Helper()
+ resp, err := ctl.issueServiceClient.CreateIssue(ctx, connect.NewRequest(&v1pb.CreateIssueRequest{
+ Parent: project.Name,
+ Issue: &v1pb.Issue{
+ Title: title,
+ Description: title + " description",
+ Type: v1pb.Issue_DATABASE_CHANGE,
+ Plan: plan.Name,
+ },
+ }))
+ require.NoError(t, err)
+ return resp.Msg
+}
+
+func approveIssueAs(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue, who testApprover) {
+ t.Helper()
+ withImpersonation(ctx, t, ctl, who, func() {
+ _, err := ctl.issueServiceClient.ApproveIssue(ctx, connect.NewRequest(&v1pb.ApproveIssueRequest{
+ Name: issue.Name,
+ }))
+ require.NoError(t, err)
+ })
+}
+
+func rejectIssueAs(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue, who testApprover, comment string) {
+ t.Helper()
+ withImpersonation(ctx, t, ctl, who, func() {
+ _, err := ctl.issueServiceClient.RejectIssue(ctx, connect.NewRequest(&v1pb.RejectIssueRequest{
+ Name: issue.Name,
+ Comment: comment,
+ }))
+ require.NoError(t, err)
+ })
+}
+
+// requestIssueAsCreator clears the rejected-approver state on a sent-back issue.
+// MUST be called as the issue creator (canRequestIssue at issue_service.go:803-805).
+// Run with the default ctl token, which is the demo user used as default creator.
+func requestIssueAsCreator(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue, comment string) {
+ t.Helper()
+ _, err := ctl.issueServiceClient.RequestIssue(ctx, connect.NewRequest(&v1pb.RequestIssueRequest{
+ Name: issue.Name,
+ Comment: comment,
+ }))
+ require.NoError(t, err)
+}
+
+// waitForApprovalFindingDone blocks until the issue's approval-finding pipeline
+// finishes — i.e., the runner has resolved the workspace approval rules and
+// the issue's ApprovalStatus is no longer CHECKING. Use when the test does not
+// care WHICH terminal status was reached, only that the runner completed (e.g.
+// when asserting that no approval webhook fires for an issue that should auto-
+// approve because no rule applies).
+func waitForApprovalFindingDone(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue) {
+ t.Helper()
+ deadline := time.Now().Add(webhookWaitTimeout)
+ for {
+ resp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{
+ Name: issue.Name,
+ }))
+ require.NoError(t, err)
+ if resp.Msg.ApprovalStatus != v1pb.Issue_CHECKING {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("issue %s still CHECKING after %s", issue.Name, webhookWaitTimeout)
+ }
+ time.Sleep(500 * time.Millisecond)
+ }
+}
+
+// waitForIssueApproved blocks until the issue reaches APPROVED.
+//
+// Use only as a state-transition post-condition. This helper is NOT a barrier
+// for webhook delivery — webhook.Manager.CreateEvent runs asynchronously after
+// the issue row is updated. The current absence-assertion call sites are
+// race-free because they subscribe only to event types the gRPC handler does
+// NOT emit on the awaited transition (e.g., I2 subscribes to ISSUE_CREATED but
+// drives an approval that emits ISSUE_APPROVED — which has no subscriber, so
+// CreateEvent's synchronous len(webhookList) == 0 short-circuit never spawns
+// a goroutine). A future caller that subscribes to the awaited event must wait
+// on the collector, not on issue state.
+func waitForIssueApproved(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue) {
+ t.Helper()
+ deadline := time.Now().Add(webhookWaitTimeout)
+ for {
+ resp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{
+ Name: issue.Name,
+ }))
+ require.NoError(t, err)
+ if resp.Msg.ApprovalStatus == v1pb.Issue_APPROVED {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("issue %s did not reach APPROVED within %s; current status %s",
+ issue.Name, webhookWaitTimeout, resp.Msg.ApprovalStatus)
+ }
+ time.Sleep(200 * time.Millisecond)
+ }
+}
+
+// waitForIssuePending blocks until the issue's approval-finding pipeline
+// finishes and the issue is in PENDING — i.e., the rule has been resolved and
+// the system is ready for an approver action. Fails the test on any other
+// terminal status (APPROVED, REJECTED), which would indicate a setup bug.
+func waitForIssuePending(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue) {
+ t.Helper()
+ deadline := time.Now().Add(webhookWaitTimeout)
+ for {
+ resp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{
+ Name: issue.Name,
+ }))
+ require.NoError(t, err)
+ switch resp.Msg.ApprovalStatus {
+ case v1pb.Issue_PENDING:
+ return
+ case v1pb.Issue_CHECKING:
+ // keep waiting
+ default:
+ t.Fatalf("issue %s reached unexpected approval status %s while waiting for PENDING",
+ issue.Name, resp.Msg.ApprovalStatus)
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("issue %s still CHECKING after %s", issue.Name, webhookWaitTimeout)
+ }
+ time.Sleep(500 * time.Millisecond)
+ }
+}
diff --git a/backend/tests/webhook_test.go b/backend/tests/webhook_test.go
index 3ab4cd7077..2f58a1fbcd 100644
--- a/backend/tests/webhook_test.go
+++ b/backend/tests/webhook_test.go
@@ -7,6 +7,7 @@ import (
"io"
"net/http"
"net/http/httptest"
+ "path"
"strings"
"sync"
"testing"
@@ -146,7 +147,15 @@ func TestWebhookIntegration(t *testing.T) {
require.NoError(t, err)
defer ctl.Close(ctx)
- // Create a test webhook server
+ // Test webhook server.
+ //
+ // Body MUST be the literal "ok" — the Slack plugin's postMessage
+ // (backend/plugin/webhook/slack/slack.go:247-249) treats any other body as
+ // a delivery failure and triggers common.Retry (up to 3 attempts, 5s
+ // exponential backoff). Without an "ok" response, every webhook event
+ // would be delivered 1-3 times depending on test timing, causing
+ // nondeterministic counts in the requireWebhookCount assertions across
+ // the trigger matrix.
collector := &webhookCollector{}
webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := collector.addRequest(r); err != nil {
@@ -154,7 +163,7 @@ func TestWebhookIntegration(t *testing.T) {
return
}
w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{"success": true}`))
+ _, _ = w.Write([]byte("ok"))
}))
defer webhookServer.Close()
@@ -176,91 +185,525 @@ func TestWebhookIntegration(t *testing.T) {
require.NoError(t, err)
instance := instanceResp.Msg
- // Create webhooks for issue events
- for _, eventType := range []v1pb.Activity_Type{
- v1pb.Activity_ISSUE_CREATED,
- } {
- _, err := ctl.projectServiceClient.AddWebhook(ctx, connect.NewRequest(&v1pb.AddWebhookRequest{
- Project: ctl.project.Name,
- Webhook: &v1pb.Webhook{
- Type: v1pb.WebhookType_SLACK,
- Title: fmt.Sprintf("Test Webhook for %s", eventType),
- Url: webhookServer.URL,
- NotificationTypes: []v1pb.Activity_Type{eventType},
- },
- }))
- require.NoError(t, err)
- }
-
t.Run("IssueWithPlanWebhookPayload", func(t *testing.T) {
- // Reset webhook collector for this test
collector.reset()
- // Create a plan with title and description
- planTitle := "Database Migration Plan"
+ project := ctl.createTestProject(ctx, t, "byt9398-i1")
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})
+
planDesc := "This plan creates a new database with important schema changes"
planResp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{
- Parent: ctl.project.Name,
+ Parent: project.Name,
Plan: &v1pb.Plan{
- Name: planTitle,
+ Name: "Database Migration Plan",
Description: planDesc,
- Specs: []*v1pb.Plan_Spec{
- {
- Id: uuid.NewString(),
- Config: &v1pb.Plan_Spec_CreateDatabaseConfig{
- CreateDatabaseConfig: &v1pb.Plan_CreateDatabaseConfig{
- Target: instance.Name,
- Database: "testdb",
- },
+ Specs: []*v1pb.Plan_Spec{{
+ Id: uuid.NewString(),
+ Config: &v1pb.Plan_Spec_CreateDatabaseConfig{
+ CreateDatabaseConfig: &v1pb.Plan_CreateDatabaseConfig{
+ Target: instance.Name,
+ Database: "testdb",
},
},
- },
+ }},
},
}))
require.NoError(t, err)
- // Create an issue for webhook testing
- issueResp, err := ctl.issueServiceClient.CreateIssue(ctx, connect.NewRequest(&v1pb.CreateIssueRequest{
- Parent: ctl.project.Name,
+ _, err = ctl.issueServiceClient.CreateIssue(ctx, connect.NewRequest(&v1pb.CreateIssueRequest{
+ Parent: project.Name,
Issue: &v1pb.Issue{
Title: "Test webhook issue",
- Description: "", // Empty description is OK
+ Description: "",
Type: v1pb.Issue_DATABASE_CHANGE,
Plan: planResp.Msg.Name,
},
}))
require.NoError(t, err)
- require.NotEmpty(t, issueResp.Msg.Name)
-
- // Wait for webhook to be processed
- time.Sleep(5 * time.Second)
- // Verify webhook was triggered
- requests := collector.getRequests()
- require.GreaterOrEqual(t, len(requests), 1, "Expected at least 1 webhook")
+ // Deterministic wait — replaces the original time.Sleep(5*time.Second).
+ waitForWebhookCount(t, collector, project.Name, "Issue created", 1)
- // Find and verify the issue creation webhook. Slack incoming webhook
- // payloads intentionally omit top-level text to avoid rendering a
- // duplicate message above the attachment card.
+ // Find the captured request and verify payload contents.
var foundCorrectWebhook bool
- for _, req := range requests {
+ for _, req := range collector.getRequests() {
require.Equal(t, "POST", req.Method)
require.Contains(t, req.Headers.Get("Content-Type"), "application/json")
- title, desc, err := parseSlackWebhook(req.Body)
+ _, desc, err := parseSlackWebhook(req.Body)
require.NoError(t, err)
-
- // The webhook should use plan's description (title support is incomplete)
- if desc == planDesc {
- foundCorrectWebhook = true
- t.Logf("✓ Webhook uses plan's description: %q", desc)
- if title == "" {
- t.Logf("⚠️ Issue title is empty (expected: %q)", planTitle)
- }
- break
+ // The webhook payload uses the plan's description (issue-title support is incomplete).
+ if desc != planDesc {
+ continue
}
+ foundCorrectWebhook = true
+
+ // I1 assertions: event identity and payload contents.
+ require.True(t, matchesEvent(req, project.Name, "Issue created"),
+ "first webhook section should contain 'Issue created'; got %q",
+ firstSlackSectionText(req.Body))
+
+ body := string(req.Body)
+ require.Contains(t, body, project.Name, "payload should reference the project resource name")
+ require.Contains(t, body, "Test webhook issue", "payload should contain the issue title")
+ require.Contains(t, body, fmt.Sprintf("/projects/%s/issues/", path.Base(project.Name)),
+ "payload should link to the project's issue resource")
+ break
}
+ require.True(t, foundCorrectWebhook, "expected a webhook whose Slack description matches the plan description")
+ })
+
+ t.Run("PipelineCompletedAfterSkippingFailedTask", func(t *testing.T) {
+ collector.reset()
+
+ project := ctl.createTestProject(ctx, t, "byt9398-c4")
+
+ // Create databases before registering the webhook so the implicit
+ // PIPELINE_COMPLETED events fired by createDatabase (which internally
+ // runs a createDatabaseConfig plan) do not pollute the collector.
+ err := ctl.createDatabase(ctx, project, instance, nil, "byt9398_c4_pass", "")
+ require.NoError(t, err)
+ err = ctl.createDatabase(ctx, project, instance, nil, "byt9398_c4_fail", "")
+ require.NoError(t, err)
+
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
+ v1pb.Activity_PIPELINE_FAILED,
+ v1pb.Activity_PIPELINE_COMPLETED,
+ })
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c4_pass")},
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c4_fail")},
+ })
+ rollout := runAllTasks(ctx, t, ctl, plan)
+
+ // Phase 1: failing task → exactly one PIPELINE_FAILED, no PIPELINE_COMPLETED.
+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)
+ requireWebhookCount(t, collector, project.Name, "Rollout completed", 0)
+
+ // Phase 2: skip the failed task → PIPELINE_COMPLETED fires (the fix).
+ skipFailedTasks(ctx, t, ctl, rollout)
+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ })
+
+ t.Run("PipelineCompleted_AllTasksDone", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-c1")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c1_a", ""))
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c1_b", ""))
+ collector.reset() // flush any PIPELINE_COMPLETED from database creation
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,
+ })
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c1_a")},
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c1_b")},
+ })
+ runAllTasks(ctx, t, ctl, plan)
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)
+ })
+
+ t.Run("PipelineCompleted_DoneAndSkipped", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-c2")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c2_a", ""))
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c2_b", ""))
+ collector.reset() // flush any PIPELINE_COMPLETED from database creation
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,
+ })
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c2_a")},
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c2_b")},
+ })
+ rollout := createRolloutOnly(ctx, t, ctl, plan)
+ skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c2_b"))
+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c2_a"))
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)
+ })
+
+ t.Run("PipelineCompleted_AllSkipped", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-c5")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c5_a", ""))
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c5_b", ""))
+ collector.reset() // flush any PIPELINE_COMPLETED from database creation
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,
+ })
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c5_a")},
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c5_b")},
+ })
+ rollout := createRolloutOnly(ctx, t, ctl, plan)
+ skipAllTasks(ctx, t, ctl, rollout)
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)
+ })
+
+ t.Run("PipelineCompleted_DoneAndFailedThenRetriedDone", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-c3")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c3_pass", ""))
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c3_fail", ""))
+ collector.reset() // flush any PIPELINE_COMPLETED from database creation
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,
+ })
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c3_pass")},
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c3_fail")},
+ })
+ rollout := runAllTasks(ctx, t, ctl, plan)
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)
+ requireWebhookCount(t, collector, project.Name, "Rollout completed", 0)
+
+ unblockFailingTask(t, instanceDir, "byt9398_c3_fail")
+ retryFailedTasks(ctx, t, ctl, rollout)
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ })
+
+ t.Run("PipelineCompleted_AllFailedThenAllSkipped", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-c6")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c6_a", ""))
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c6_b", ""))
+ collector.reset() // flush any PIPELINE_COMPLETED from database creation
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,
+ })
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c6_a")},
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c6_b")},
+ })
+ rollout := runAllTasks(ctx, t, ctl, plan)
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)
+
+ skipAllTasks(ctx, t, ctl, rollout)
+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ })
+
+ t.Run("PipelineCompleted_MixedRecovery", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-c7")
+ for _, n := range []string{"byt9398_c7_done", "byt9398_c7_skip", "byt9398_c7_retry", "byt9398_c7_skipfailed"} {
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, n, ""))
+ }
+ collector.reset() // flush any PIPELINE_COMPLETED from database creation
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,
+ })
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_done")},
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_skip")},
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_retry")},
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_skipfailed")},
+ })
+
+ rollout := createRolloutOnly(ctx, t, ctl, plan)
+ skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skip"))
+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_done"))
+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"))
+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skipfailed"))
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 60*time.Second)
+ requireWebhookCount(t, collector, project.Name, "Rollout completed", 0)
+
+ // Unblock dbRetry only — dbSkipFailed's __force_fail_target table remains
+ // absent in its own .db file, so its retry would still fail.
+ unblockFailingTask(t, instanceDir, "byt9398_c7_retry")
+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"))
+ waitForTaskStatus(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"), v1pb.Task_DONE, 30*time.Second)
+ skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skipfailed"))
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)
+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ })
+
+ t.Run("PipelineFailed_SingleTaskFails", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-f1")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f1_fail", ""))
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f1_fail")},
+ })
+ runAllTasks(ctx, t, ctl, plan)
+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ })
+
+ t.Run("PipelineFailed_DedupOnSecondTaskFailure", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-f2")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f2_a", ""))
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f2_b", ""))
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f2_a")},
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f2_b")},
+ })
+ rollout := runAllTasks(ctx, t, ctl, plan)
- require.True(t, foundCorrectWebhook, "Webhook should use plan's description")
+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)
+
+ // Both tasks have failed. ClaimPipelineFailureNotification's PK collision
+ // must dedupe — assert exactly 1 even after both terminal.
+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ })
+
+ t.Run("PipelineFailed_RetryFailsAgain", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-f3")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f3_fail", ""))
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f3_fail")},
+ })
+ rollout := runAllTasks(ctx, t, ctl, plan)
+
+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)
+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)
+
+ // BatchRunTasks resets the dedup row before enqueuing the retry, so the
+ // second failure must re-fire PIPELINE_FAILED. We deliberately do NOT call
+ // unblockFailingTask — the retry should fail again.
+ retryFailedTasks(ctx, t, ctl, rollout)
+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 2)
+ })
+
+ t.Run("IssueCreated_NoLeakageFromOtherEvents", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-i2")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_i2_db", ""))
+
+ disableSelfApproval(ctx, t, ctl, project)
+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
+ appr := provisionApprover(ctx, t, ctl, project, "i2", "roles/projectOwner")
+
+ collector.reset() // flush any creation-time webhook events
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_i2_db")},
+ })
+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "I2 issue")
+
+ waitForWebhookCount(t, collector, project.Name, "Issue created", 1)
+ waitForIssuePending(ctx, t, ctl, issue)
+
+ // Drive an approval. The webhook is NOT subscribed to ISSUE_APPROVED, so the
+ // count must stay at 1. Wait for the issue to actually reach APPROVED before
+ // asserting — that proves the approval pipeline ran to completion and any
+ // approval-related webhooks have already had their chance to fire.
+ approveIssueAs(ctx, t, ctl, issue, appr)
+ waitForIssueApproved(ctx, t, ctl, issue)
+ requireWebhookCount(t, collector, project.Name, "Issue created", 1)
+ requireWebhookCount(t, collector, project.Name, "Issue approved", 0)
+ })
+
+ t.Run("IssueApprovalRequested_FiresWhenRequired", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-a1")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_a1_db", ""))
+
+ disableSelfApproval(ctx, t, ctl, project)
+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
+ _ = provisionApprover(ctx, t, ctl, project, "a1", "roles/projectOwner")
+
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVAL_REQUESTED})
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_a1_db")},
+ })
+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "A1 issue")
+ waitForIssuePending(ctx, t, ctl, issue)
+
+ waitForWebhookCount(t, collector, project.Name, "Approval required", 1)
+ })
+
+ t.Run("IssueApprovalRequested_NotFiredWhenUnused", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-a2")
+ // The workspace has a default catch-all approval rule. Clear it so that
+ // no rule applies to A2's project, then assert no ISSUE_APPROVAL_REQUESTED fires.
+ clearWorkspaceApprovalRules(ctx, t, ctl)
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_a2_db", ""))
+
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVAL_REQUESTED})
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_a2_db")},
+ })
+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "A2 issue")
+
+ // Wait for the approval-finding pipeline to finish (any terminal status —
+ // with no rule applying, the issue auto-resolves). After this wait the
+ // runner has had its chance to fire any approval-related webhook, so the
+ // absence assertion is no longer racing the runner.
+ waitForApprovalFindingDone(ctx, t, ctl, issue)
+ requireWebhookCount(t, collector, project.Name, "Approval required", 0)
+ })
+
+ t.Run("IssueApproved_SingleStep", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-ap1")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_ap1_db", ""))
+
+ clearWorkspaceApprovalRules(ctx, t, ctl)
+ disableSelfApproval(ctx, t, ctl, project)
+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
+ appr := provisionApprover(ctx, t, ctl, project, "ap1", "roles/projectOwner")
+
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVED})
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_ap1_db")},
+ })
+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "AP1 issue")
+ waitForIssuePending(ctx, t, ctl, issue)
+
+ approveIssueAs(ctx, t, ctl, issue, appr)
+ waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)
+ })
+
+ t.Run("IssueApproved_MultiStepOnlyFiresAtFinal", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-ap2")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_ap2_db", ""))
+
+ clearWorkspaceApprovalRules(ctx, t, ctl)
+ disableSelfApproval(ctx, t, ctl, project)
+ installWorkspaceApprovalRule(ctx, t, ctl, project,
+ []string{"roles/projectDeveloper", "roles/projectOwner"})
+ apprStep1 := provisionApprover(ctx, t, ctl, project, "ap2-step1", "roles/projectDeveloper")
+ apprStep2 := provisionApprover(ctx, t, ctl, project, "ap2-step2", "roles/projectOwner")
+
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVED})
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_ap2_db")},
+ })
+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "AP2 issue")
+ waitForIssuePending(ctx, t, ctl, issue)
+
+ // After step-1 approval the issue must remain PENDING (step 2 still required).
+ // waitForIssuePending is a stable post-condition — by the time it returns the
+ // approval runner has updated the next-step approver state and the gRPC
+ // handler has already decided not to call NotifyIssueApproved (because
+ // approved == false). No goroutine for ISSUE_APPROVED was ever spawned, so
+ // the count assertion below is race-free.
+ approveIssueAs(ctx, t, ctl, issue, apprStep1)
+ waitForIssuePending(ctx, t, ctl, issue)
+ requireWebhookCount(t, collector, project.Name, "Issue approved", 0)
+
+ approveIssueAs(ctx, t, ctl, issue, apprStep2)
+ waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)
+ })
+
+ t.Run("IssueSentBack_FiresOnRejection", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-sb1")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_sb1_db", ""))
+
+ clearWorkspaceApprovalRules(ctx, t, ctl)
+ disableSelfApproval(ctx, t, ctl, project)
+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
+ appr := provisionApprover(ctx, t, ctl, project, "sb1", "roles/projectOwner")
+
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_SENT_BACK})
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_sb1_db")},
+ })
+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "SB1 issue")
+ waitForIssuePending(ctx, t, ctl, issue)
+
+ rejectIssueAs(ctx, t, ctl, issue, appr, "needs more context")
+ waitForWebhookCount(t, collector, project.Name, "Issue sent back", 1)
+ })
+
+ t.Run("IssueSentBack_ThenReapproved_E2E", func(t *testing.T) {
+ collector.reset()
+ project := ctl.createTestProject(ctx, t, "byt9398-sb2")
+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_sb2_db", ""))
+
+ clearWorkspaceApprovalRules(ctx, t, ctl)
+ disableSelfApproval(ctx, t, ctl, project)
+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})
+ appr := provisionApprover(ctx, t, ctl, project, "sb2", "roles/projectOwner")
+
+ collector.reset()
+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{
+ v1pb.Activity_ISSUE_SENT_BACK,
+ v1pb.Activity_ISSUE_APPROVED,
+ v1pb.Activity_ISSUE_APPROVAL_REQUESTED,
+ })
+
+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{
+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_sb2_db")},
+ })
+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "SB2 issue")
+ waitForIssuePending(ctx, t, ctl, issue)
+
+ rejectIssueAs(ctx, t, ctl, issue, appr, "fix and resubmit")
+ waitForWebhookCount(t, collector, project.Name, "Issue sent back", 1)
+
+ // Issue creator (default ctl token) re-requests approval. Note: RequestIssue
+ // does NOT reset ApprovalFindingDone, so waitForIssuePending would return
+ // immediately. Wait for the second "Approval required" webhook instead —
+ // RequestIssue calls approval.NotifyApprovalRequested directly.
+ requestIssueAsCreator(ctx, t, ctl, issue, "addressed feedback")
+ waitForWebhookCount(t, collector, project.Name, "Approval required", 2)
+
+ approveIssueAs(ctx, t, ctl, issue, appr)
+ waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)
+ })
+
+ // Regression for the second Codex P1 finding on PR #20212. Empty BatchSkipTasks
+ // must be rejected at the API boundary, otherwise it sneaks past the
+ // task-parse, environmentSet permission, and store mutation paths and would
+ // run ResetPlanWebhookDelivery on a request that did nothing — re-arming the
+ // dedup ledger for any plan the caller can name.
+ t.Run("BatchSkipTasks_RejectsEmptyTasks", func(t *testing.T) {
+ project := ctl.createTestProject(ctx, t, "byt9398-empty-skip")
+ // Parent format doesn't matter here; the empty-tasks guard fires first.
+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{
+ Parent: project.Name + "/plans/-/rollouts/-/stages/-",
+ Tasks: []string{},
+ }))
+ require.Error(t, err)
+ require.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err),
+ "BatchSkipTasks must reject empty Tasks at the API boundary")
})
}