Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(rollout): emit PIPELINE_COMPLETED after BatchSkipTasks resolves a failure (BYT-9398) (#20212)
zchpeter35c30da829d2
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.goindex 6550f24887..ba63b0e576 100644--- a/backend/api/v1/rollout_service.go+++ b/backend/api/v1/rollout_service.go@@ -741,12 +741,6 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Requestreturn nil, connect.NewError(connect.CodeNotFound, errors.Errorf("rollout (plan) %v not found", planID))}- // Reset notification state so user gets fresh feedback on retry- if err := s.store.ResetPlanWebhookDelivery(ctx, projectID, planID); err != nil {- slog.Error("failed to reset plan webhook delivery", log.BBError(err))- // Don't fail the request - notification is non-critical- }-issueN, err := s.store.GetIssue(ctx, &store.FindIssueMessage{Workspace: common.GetWorkspaceIDFromContext(ctx),ProjectIDs: []string{projectID},@@ -844,6 +838,21 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request}})+ // Reset notification state so the user gets fresh feedback on this retry.+ // Placed AFTER all validation (auth, approval, task parse) so a rejected+ // request leaves the dedup ledger untouched, but BEFORE+ // CreatePendingTaskRuns so the pending-task scheduler — which polls+ // independently on its own ticker — cannot observe the new task_run rows+ // before the stale plan_webhook_delivery row is cleared. Without this+ // ordering, a fast-failing retry could be picked up, fail, and try to+ // claim PIPELINE_FAILED against the stale row (ON CONFLICT DO NOTHING),+ // and the API's later reset would clear the row with no pending event+ // left to re-fire. Errors are logged and swallowed so a transient DB+ // hiccup doesn't fail the user-facing run request.+ if err := s.store.ResetPlanWebhookDelivery(ctx, projectID, planID); err != nil {+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))+ }+if err := s.store.CreatePendingTaskRuns(ctx, user.Email, taskRunCreates...); err != nil {return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create pending task runs, error %v", err))}@@ -857,6 +866,14 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request// BatchSkipTasks skips tasks in batch.func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Request[v1pb.BatchSkipTasksRequest]) (*connect.Response[v1pb.BatchSkipTasksResponse], error) {request := req.Msg+ // Reject empty input early. Without this guard the request would skip the+ // task-parse loop, the environmentSet-based permission check, and arrive+ // at a no-op store.BatchSkipTasks — but still run ResetPlanWebhookDelivery+ // and signal the completion check, mutating dedup state for a request that+ // did nothing. Mirrors the BatchRunTasks pattern.+ if len(request.Tasks) == 0 {+ return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("The tasks in request cannot be empty"))+ }projectID, planID, _, err := common.GetProjectIDPlanIDMaybeStageID(request.Parent)if err != nil {return nil, connect.NewError(connect.CodeInvalidArgument, err)@@ -936,6 +953,17 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Requesreturn nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to skip tasks"))}+ // Reset notification state so PIPELINE_COMPLETED can fire after skipping a+ // failed task (the BYT-9398 fix). Placed after authorization + the+ // successful BatchSkipTasks so a rejected request (auth fail, unknown+ // task, etc.) leaves the dedup ledger untouched. Must run before the+ // completion-check signal below so the async claim sees the cleared row.+ // Errors are logged and swallowed so a DB hiccup doesn't fail the+ // user-facing skip request.+ if err := s.store.ResetPlanWebhookDelivery(ctx, projectID, planID); err != nil {+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))+ }+// Signal to check if plan is complete and successful (may send PIPELINE_COMPLETED)s.bus.PlanCompletionCheckChan <- bus.PlanRef{ProjectID: projectID, PlanID: planID}diff --git a/backend/tests/webhook_helpers_test.go b/backend/tests/webhook_helpers_test.gonew file mode 100644index 0000000000..75ebe744b6--- /dev/null+++ b/backend/tests/webhook_helpers_test.go@@ -0,0 +1,705 @@+package tests++import (+ "context"+ "database/sql"+ "encoding/json"+ "fmt"+ "path"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "connectrpc.com/connect"+ "github.com/google/uuid"+ "github.com/stretchr/testify/require"+ "google.golang.org/genproto/googleapis/type/expr"+ "google.golang.org/protobuf/types/known/fieldmaskpb"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// firstSlackSectionText returns the text of the first Slack section block in the+// captured payload, or "" if none. attachments[0].blocks[0].text.text holds the+// event title decorated with emoji and Slack <link|title> markup — see+// backend/plugin/webhook/slack/slack_test.go:42-47, 85.+func firstSlackSectionText(body []byte) string {+ var payload map[string]any+ if err := json.Unmarshal(body, &payload); err != nil {+ return ""+ }+ attachments, ok := payload["attachments"].([]any)+ if !ok || len(attachments) == 0 {+ return ""+ }+ att, ok := attachments[0].(map[string]any)+ if !ok {+ return ""+ }+ blocks, ok := att["blocks"].([]any)+ if !ok || len(blocks) == 0 {+ return ""+ }+ block0, ok := blocks[0].(map[string]any)+ if !ok || block0["type"] != "section" {+ return ""+ }+ textMap, ok := block0["text"].(map[string]any)+ if !ok {+ return ""+ }+ text, ok := textMap["text"].(string)+ if !ok {+ return ""+ }+ return text+}++// matchesEvent reports whether a captured Slack payload matches the given+// project and event title. Project filtering prevents cross-subtest+// contamination from the async webhook dispatcher.+func matchesEvent(req webhookRequest, projectName, eventTitle string) bool {+ body := string(req.Body)+ if !strings.Contains(body, projectName) {+ return false+ }+ return strings.Contains(firstSlackSectionText(req.Body), eventTitle)+}++// webhookWaitTimeout is the deadline for waitForWebhookCount and the issue-state+// wait helpers. Webhook delivery + completion-check fanout takes <5s in practice;+// 30s gives ample headroom under CI load.+const webhookWaitTimeout = 30 * time.Second++// waitForWebhookCount blocks until at least n webhooks for (project, eventTitle)+// have arrived, or fails the test after webhookWaitTimeout.+func waitForWebhookCount(t *testing.T, c *webhookCollector, projectName, eventTitle string, n int) {+ t.Helper()+ deadline := time.Now().Add(webhookWaitTimeout)+ for {+ count := countWebhooksFor(c, projectName, eventTitle)+ if count >= n {+ return+ }+ if time.Now().After(deadline) {+ t.Fatalf("timed out waiting for %d %q webhooks on %s; got %d after %s",+ n, eventTitle, projectName, count, webhookWaitTimeout)+ }+ time.Sleep(100 * time.Millisecond)+ }+}++// requireWebhookCount asserts the exact count of webhooks for (project, eventTitle).+func requireWebhookCount(t *testing.T, c *webhookCollector, projectName, eventTitle string, n int) {+ t.Helper()+ got := countWebhooksFor(c, projectName, eventTitle)+ require.Equalf(t, n, got, "expected %d %q webhooks on %s, got %d", n, eventTitle, projectName, got)+}++func countWebhooksFor(c *webhookCollector, projectName, eventTitle string) int {+ n := 0+ for _, req := range c.getRequests() {+ if matchesEvent(req, projectName, eventTitle) {+ n+++ }+ }+ return n+}++// createTestProject creates a fresh project (default AllowSelfApproval=true;+// approval-flow subtests call disableSelfApproval).+func (ctl *controller) createTestProject(ctx context.Context, t *testing.T, prefix string) *v1pb.Project {+ t.Helper()+ pid := generateRandomString(prefix)+ resp, err := ctl.projectServiceClient.CreateProject(ctx, connect.NewRequest(&v1pb.CreateProjectRequest{+ ProjectId: pid,+ Project: &v1pb.Project{+ Name: fmt.Sprintf("projects/%s", pid),+ Title: prefix,+ AllowSelfApproval: true,+ },+ }))+ require.NoError(t, err)+ return resp.Msg+}++// addWebhookForEvents adds a Slack webhook to the project subscribed to the+// given event types only.+func addWebhookForEvents(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, url string, events []v1pb.Activity_Type) {+ t.Helper()+ _, err := ctl.projectServiceClient.AddWebhook(ctx, connect.NewRequest(&v1pb.AddWebhookRequest{+ Project: project.Name,+ Webhook: &v1pb.Webhook{+ Type: v1pb.WebhookType_SLACK,+ Title: "test-webhook-" + generateRandomString("hook"),+ Url: url,+ NotificationTypes: events,+ },+ }))+ require.NoError(t, err)+}++// dbTargetName returns the canonical instance/databases/<name> resource form.+func dbTargetName(instance *v1pb.Instance, dbName string) string {+ return fmt.Sprintf("%s/databases/%s", instance.Name, dbName)+}++// seedPassingSheet creates a sheet whose SQL trivially succeeds.+func seedPassingSheet(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project) string {+ t.Helper()+ resp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{+ Parent: project.Name,+ Sheet: &v1pb.Sheet{Content: []byte("SELECT 1;")},+ }))+ require.NoError(t, err)+ return resp.Msg.Name+}++// seedFailingSheet creates a sheet whose SQL fails until unblockFailingTask runs.+func seedFailingSheet(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project) string {+ t.Helper()+ resp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{+ Parent: project.Name,+ Sheet: &v1pb.Sheet{Content: []byte("INSERT INTO __force_fail_target VALUES(1);")},+ }))+ require.NoError(t, err)+ return resp.Msg.Name+}++// unblockFailingTask creates the missing table inside the SQLite database file+// so subsequent runs of seedFailingSheet's SQL succeed. db.Close() flushes+// WAL/journal so the file is consistent before the retry is enqueued.+func unblockFailingTask(t *testing.T, instanceDir, dbName string) {+ t.Helper()+ dbPath := filepath.Join(instanceDir, dbName+".db")+ db, err := sql.Open("sqlite3", dbPath)+ require.NoError(t, err)+ defer func() {+ if cerr := db.Close(); cerr != nil {+ t.Errorf("close sqlite handle for %s: %v", dbPath, cerr)+ }+ }()+ _, err = db.Exec("CREATE TABLE IF NOT EXISTS __force_fail_target(id INT);")+ require.NoError(t, err)+}++type taskSpec struct {+ sheetName string+ dbTarget string // full instance/databases/<name>+}++// createPlanWithSpecs creates a plan with one ChangeDatabaseConfig spec per entry.+func createPlanWithSpecs(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, specs []taskSpec) *v1pb.Plan {+ t.Helper()+ var planSpecs []*v1pb.Plan_Spec+ for _, s := range specs {+ planSpecs = append(planSpecs, &v1pb.Plan_Spec{+ Id: uuid.NewString(),+ Config: &v1pb.Plan_Spec_ChangeDatabaseConfig{+ ChangeDatabaseConfig: &v1pb.Plan_ChangeDatabaseConfig{+ Targets: []string{s.dbTarget},+ Sheet: s.sheetName,+ },+ },+ })+ }+ resp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{+ Parent: project.Name,+ Plan: &v1pb.Plan{Specs: planSpecs},+ }))+ require.NoError(t, err)+ return resp.Msg+}++func createRolloutOnly(ctx context.Context, t *testing.T, ctl *controller, plan *v1pb.Plan) *v1pb.Rollout {+ t.Helper()+ resp, err := ctl.rolloutServiceClient.CreateRollout(ctx, connect.NewRequest(&v1pb.CreateRolloutRequest{Parent: plan.Name}))+ require.NoError(t, err)+ return resp.Msg+}++// runAllTasks materializes the rollout and starts every task in every stage.+func runAllTasks(ctx context.Context, t *testing.T, ctl *controller, plan *v1pb.Plan) *v1pb.Rollout {+ t.Helper()+ rollout := createRolloutOnly(ctx, t, ctl, plan)+ for _, stage := range rollout.Stages {+ var taskNames []string+ for _, task := range stage.Tasks {+ taskNames = append(taskNames, task.Name)+ }+ if len(taskNames) == 0 {+ continue+ }+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{+ Parent: stage.Name,+ Tasks: taskNames,+ }))+ require.NoError(t, err)+ }+ return rollout+}++func refreshRollout(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) *v1pb.Rollout {+ t.Helper()+ resp, err := ctl.rolloutServiceClient.GetRollout(ctx, connect.NewRequest(&v1pb.GetRolloutRequest{Name: rollout.Name}))+ require.NoError(t, err)+ return resp.Msg+}++// findTaskByDB scans the rollout for a task targeting dbResource.+func findTaskByDB(t *testing.T, rollout *v1pb.Rollout, dbResource string) (stageName, taskName string) {+ t.Helper()+ for _, stage := range rollout.Stages {+ for _, task := range stage.Tasks {+ if task.Target == dbResource {+ return stage.Name, task.Name+ }+ }+ }+ t.Fatalf("no task targets database %s in rollout %s", dbResource, rollout.Name)+ return "", ""+}++func runTaskByDB(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout, dbResource string) {+ t.Helper()+ stageName, taskName := findTaskByDB(t, rollout, dbResource)+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{+ Parent: stageName,+ Tasks: []string{taskName},+ }))+ require.NoError(t, err)+}++func skipTaskByDB(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout, dbResource string) {+ t.Helper()+ stageName, taskName := findTaskByDB(t, rollout, dbResource)+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{+ Parent: stageName,+ Tasks: []string{taskName},+ Reason: "test: skip task by db",+ }))+ require.NoError(t, err)+}++// skipFailedTasks finds every FAILED task in the rollout and skips them.+func skipFailedTasks(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) {+ t.Helper()+ fresh := refreshRollout(ctx, t, ctl, rollout)+ perStage := map[string][]string{}+ for _, stage := range fresh.Stages {+ for _, task := range stage.Tasks {+ if task.Status == v1pb.Task_FAILED {+ perStage[stage.Name] = append(perStage[stage.Name], task.Name)+ }+ }+ }+ require.NotEmpty(t, perStage, "expected at least one failed task to skip")+ for stageName, names := range perStage {+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{+ Parent: stageName,+ Tasks: names,+ Reason: "test: skip failed tasks",+ }))+ require.NoError(t, err)+ }+}++func skipAllTasks(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) {+ t.Helper()+ fresh := refreshRollout(ctx, t, ctl, rollout)+ for _, stage := range fresh.Stages {+ var names []string+ for _, task := range stage.Tasks {+ names = append(names, task.Name)+ }+ if len(names) == 0 {+ continue+ }+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{+ Parent: stage.Name,+ Tasks: names,+ Reason: "test: skip all",+ }))+ require.NoError(t, err)+ }+}++// retryFailedTasks reruns BatchRunTasks on every FAILED task. Caller decides+// whether to call unblockFailingTask first (to make the retry pass) or not+// (to test that BatchRunTasks resets the dedup row so a second+// PIPELINE_FAILED can fire).+func retryFailedTasks(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout) {+ t.Helper()+ fresh := refreshRollout(ctx, t, ctl, rollout)+ perStage := map[string][]string{}+ for _, stage := range fresh.Stages {+ for _, task := range stage.Tasks {+ if task.Status == v1pb.Task_FAILED {+ perStage[stage.Name] = append(perStage[stage.Name], task.Name)+ }+ }+ }+ require.NotEmpty(t, perStage)+ for stageName, names := range perStage {+ _, err := ctl.rolloutServiceClient.BatchRunTasks(ctx, connect.NewRequest(&v1pb.BatchRunTasksRequest{+ Parent: stageName,+ Tasks: names,+ }))+ require.NoError(t, err)+ }+}++// waitForTaskStatus polls the rollout until the task targeting dbResource has+// the requested status, or fails the test.+func waitForTaskStatus(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout, dbResource string, want v1pb.Task_Status, timeout time.Duration) {+ t.Helper()+ deadline := time.Now().Add(timeout)+ for {+ fresh := refreshRollout(ctx, t, ctl, rollout)+ for _, stage := range fresh.Stages {+ for _, task := range stage.Tasks {+ if task.Target != dbResource {+ continue+ }+ if task.Status == want {+ return+ }+ }+ }+ if time.Now().After(deadline) {+ t.Fatalf("timed out waiting for task on %s to reach %s", dbResource, want)+ }+ time.Sleep(200 * time.Millisecond)+ }+}++// waitForAllTasksTerminal polls until every task in the rollout is in a+// terminal status (DONE, FAILED, SKIPPED, or CANCELED). Used before+// "skip remaining" actions to avoid races with still-running tasks. On+// timeout, dumps per-task status to t.Logf so failures are debuggable+// without re-running.+func waitForAllTasksTerminal(ctx context.Context, t *testing.T, ctl *controller, rollout *v1pb.Rollout, timeout time.Duration) {+ t.Helper()+ deadline := time.Now().Add(timeout)+ isTerminal := func(s v1pb.Task_Status) bool {+ return s == v1pb.Task_DONE || s == v1pb.Task_FAILED || s == v1pb.Task_SKIPPED || s == v1pb.Task_CANCELED+ }+ var fresh *v1pb.Rollout+ for {+ fresh = refreshRollout(ctx, t, ctl, rollout)+ allTerminal := true+ for _, stage := range fresh.Stages {+ for _, task := range stage.Tasks {+ if !isTerminal(task.Status) {+ allTerminal = false+ }+ }+ }+ if allTerminal {+ return+ }+ if time.Now().After(deadline) {+ for _, stage := range fresh.Stages {+ for _, task := range stage.Tasks {+ t.Logf("non-terminal task: %s status=%s target=%s", task.Name, task.Status, task.Target)+ }+ }+ t.Fatalf("timed out waiting for all tasks terminal on rollout %s after %s", rollout.Name, timeout)+ }+ time.Sleep(200 * time.Millisecond)+ }+}++// installWorkspaceApprovalRule writes a WORKSPACE_APPROVAL setting rule that+// scopes to the given project via CEL and registers a cleanup that restores+// the prior workspace setting. flowRoles is the ordered list of roles for+// sequential approval steps (one role per step).+func installWorkspaceApprovalRule(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, flowRoles []string) {+ t.Helper()+ projectID := path.Base(project.Name)++ prior := snapshotWorkspaceApproval(ctx, t, ctl)++ rules := []*v1pb.WorkspaceApprovalSetting_Rule{{+ Source: v1pb.WorkspaceApprovalSetting_Rule_SOURCE_UNSPECIFIED,+ Condition: &expr.Expr{+ Expression: fmt.Sprintf(`resource.project_id == "%s"`, projectID),+ },+ Template: &v1pb.ApprovalTemplate{+ Title: "Test approval flow for " + projectID,+ Flow: &v1pb.ApprovalFlow{Roles: flowRoles},+ },+ }}+ if prior != nil {+ // Append to the snapshotted rules. Each subtest's t.Cleanup restores+ // its own snapshot, so LIFO ordering of cleanups still ends with the+ // truly-original workspace state. If a future test installs twice in+ // the same subtest, cleanups run LIFO and each writes the snapshot+ // taken at its install time, so the final state matches the original.+ rules = append(prior.Rules, rules...)+ }++ writeWorkspaceApproval(ctx, t, ctl, &v1pb.WorkspaceApprovalSetting{Rules: rules})+ t.Cleanup(func() {+ restoreWorkspaceApproval(t, ctl, prior)+ })+}++// clearWorkspaceApprovalRules removes all workspace approval rules (including the+// default catch-all) and restores the prior setting on cleanup. Use this in+// subtests that assert ISSUE_APPROVAL_REQUESTED does NOT fire when truly no rule+// applies, or when a custom multi-step flow must not coexist with the default+// projectOwner-only rule.+func clearWorkspaceApprovalRules(ctx context.Context, t *testing.T, ctl *controller) {+ t.Helper()++ prior := snapshotWorkspaceApproval(ctx, t, ctl)+ writeWorkspaceApproval(ctx, t, ctl, &v1pb.WorkspaceApprovalSetting{})+ t.Cleanup(func() {+ restoreWorkspaceApproval(t, ctl, prior)+ })+}++// snapshotWorkspaceApproval reads the current WORKSPACE_APPROVAL setting and+// fails the test if the read errors — silent failures here would let a later+// "restore" wipe the workspace's real config.+func snapshotWorkspaceApproval(ctx context.Context, t *testing.T, ctl *controller) *v1pb.WorkspaceApprovalSetting {+ t.Helper()+ resp, err := ctl.settingServiceClient.GetSetting(ctx, connect.NewRequest(&v1pb.GetSettingRequest{+ Name: "settings/WORKSPACE_APPROVAL",+ }))+ require.NoError(t, err, "snapshot WORKSPACE_APPROVAL setting")+ return resp.Msg.Value.GetWorkspaceApproval() // may be nil when no rules are set+}++func writeWorkspaceApproval(ctx context.Context, t *testing.T, ctl *controller, value *v1pb.WorkspaceApprovalSetting) {+ t.Helper()+ _, err := ctl.settingServiceClient.UpdateSetting(ctx, connect.NewRequest(&v1pb.UpdateSettingRequest{+ AllowMissing: true,+ Setting: &v1pb.Setting{+ Name: "settings/WORKSPACE_APPROVAL",+ Value: &v1pb.SettingValue{+ Value: &v1pb.SettingValue_WorkspaceApproval{WorkspaceApproval: value},+ },+ },+ }))+ require.NoError(t, err, "write WORKSPACE_APPROVAL setting")+}++// restoreWorkspaceApproval writes the snapshotted setting back. Runs from+// t.Cleanup; logs failures loudly so a corrupted teardown doesn't bleed into+// later subtests as an order-dependent flake.+func restoreWorkspaceApproval(t *testing.T, ctl *controller, prior *v1pb.WorkspaceApprovalSetting) {+ t.Helper()+ restored := prior+ if restored == nil {+ restored = &v1pb.WorkspaceApprovalSetting{}+ }+ if _, err := ctl.settingServiceClient.UpdateSetting(context.Background(), connect.NewRequest(&v1pb.UpdateSettingRequest{+ AllowMissing: true,+ Setting: &v1pb.Setting{+ Name: "settings/WORKSPACE_APPROVAL",+ Value: &v1pb.SettingValue{+ Value: &v1pb.SettingValue_WorkspaceApproval{WorkspaceApproval: restored},+ },+ },+ })); err != nil {+ t.Errorf("failed to restore WORKSPACE_APPROVAL setting in cleanup: %v", err)+ }+}++func disableSelfApproval(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project) {+ t.Helper()+ _, err := ctl.projectServiceClient.UpdateProject(ctx, connect.NewRequest(&v1pb.UpdateProjectRequest{+ Project: &v1pb.Project{+ Name: project.Name,+ AllowSelfApproval: false,+ },+ UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"allow_self_approval"}},+ }))+ require.NoError(t, err)+}++type testApprover struct {+ Email string+ Password string+}++func provisionApprover(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, suffix, projectRole string) testApprover {+ t.Helper()+ email := fmt.Sprintf("approver-%[email protected]", suffix)+ password := "1024bytebase"++ newUser, err := ctl.userServiceClient.CreateUser(ctx, connect.NewRequest(&v1pb.CreateUserRequest{+ User: &v1pb.User{Email: email, Password: password, Title: "Approver " + suffix},+ }))+ require.NoError(t, err)++ _, err = ctl.addMemberToWorkspaceIAM(ctx, newUser.Msg.Workspace, fmt.Sprintf("user:%s", email), "roles/workspaceMember")+ require.NoError(t, err)++ policyResp, err := ctl.projectServiceClient.GetIamPolicy(ctx, connect.NewRequest(&v1pb.GetIamPolicyRequest{+ Resource: project.Name,+ }))+ require.NoError(t, err)+ policy := policyResp.Msg+ policy.Bindings = append(policy.Bindings, &v1pb.Binding{+ Role: projectRole,+ Members: []string{fmt.Sprintf("user:%s", email)},+ })+ _, err = ctl.projectServiceClient.SetIamPolicy(ctx, connect.NewRequest(&v1pb.SetIamPolicyRequest{+ Resource: project.Name,+ Policy: policy,+ }))+ require.NoError(t, err)++ return testApprover{Email: email, Password: password}+}++// withImpersonation logs in as the given approver, runs fn with that identity,+// and restores the original token via defer.+func withImpersonation(ctx context.Context, t *testing.T, ctl *controller, who testApprover, fn func()) {+ t.Helper()+ loginResp, err := ctl.authServiceClient.Login(ctx, connect.NewRequest(&v1pb.LoginRequest{+ Email: who.Email,+ Password: who.Password,+ }))+ require.NoError(t, err)++ original := ctl.authInterceptor.token+ ctl.authInterceptor.token = loginResp.Msg.Token+ defer func() { ctl.authInterceptor.token = original }()++ fn()+}++func createIssueForPlan(ctx context.Context, t *testing.T, ctl *controller, project *v1pb.Project, plan *v1pb.Plan, title string) *v1pb.Issue {+ t.Helper()+ resp, err := ctl.issueServiceClient.CreateIssue(ctx, connect.NewRequest(&v1pb.CreateIssueRequest{+ Parent: project.Name,+ Issue: &v1pb.Issue{+ Title: title,+ Description: title + " description",+ Type: v1pb.Issue_DATABASE_CHANGE,+ Plan: plan.Name,+ },+ }))+ require.NoError(t, err)+ return resp.Msg+}++func approveIssueAs(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue, who testApprover) {+ t.Helper()+ withImpersonation(ctx, t, ctl, who, func() {+ _, err := ctl.issueServiceClient.ApproveIssue(ctx, connect.NewRequest(&v1pb.ApproveIssueRequest{+ Name: issue.Name,+ }))+ require.NoError(t, err)+ })+}++func rejectIssueAs(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue, who testApprover, comment string) {+ t.Helper()+ withImpersonation(ctx, t, ctl, who, func() {+ _, err := ctl.issueServiceClient.RejectIssue(ctx, connect.NewRequest(&v1pb.RejectIssueRequest{+ Name: issue.Name,+ Comment: comment,+ }))+ require.NoError(t, err)+ })+}++// requestIssueAsCreator clears the rejected-approver state on a sent-back issue.+// MUST be called as the issue creator (canRequestIssue at issue_service.go:803-805).+// Run with the default ctl token, which is the demo user used as default creator.+func requestIssueAsCreator(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue, comment string) {+ t.Helper()+ _, err := ctl.issueServiceClient.RequestIssue(ctx, connect.NewRequest(&v1pb.RequestIssueRequest{+ Name: issue.Name,+ Comment: comment,+ }))+ require.NoError(t, err)+}++// waitForApprovalFindingDone blocks until the issue's approval-finding pipeline+// finishes — i.e., the runner has resolved the workspace approval rules and+// the issue's ApprovalStatus is no longer CHECKING. Use when the test does not+// care WHICH terminal status was reached, only that the runner completed (e.g.+// when asserting that no approval webhook fires for an issue that should auto-+// approve because no rule applies).+func waitForApprovalFindingDone(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue) {+ t.Helper()+ deadline := time.Now().Add(webhookWaitTimeout)+ for {+ resp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{+ Name: issue.Name,+ }))+ require.NoError(t, err)+ if resp.Msg.ApprovalStatus != v1pb.Issue_CHECKING {+ return+ }+ if time.Now().After(deadline) {+ t.Fatalf("issue %s still CHECKING after %s", issue.Name, webhookWaitTimeout)+ }+ time.Sleep(500 * time.Millisecond)+ }+}++// waitForIssueApproved blocks until the issue reaches APPROVED.+//+// Use only as a state-transition post-condition. This helper is NOT a barrier+// for webhook delivery — webhook.Manager.CreateEvent runs asynchronously after+// the issue row is updated. The current absence-assertion call sites are+// race-free because they subscribe only to event types the gRPC handler does+// NOT emit on the awaited transition (e.g., I2 subscribes to ISSUE_CREATED but+// drives an approval that emits ISSUE_APPROVED — which has no subscriber, so+// CreateEvent's synchronous len(webhookList) == 0 short-circuit never spawns+// a goroutine). A future caller that subscribes to the awaited event must wait+// on the collector, not on issue state.+func waitForIssueApproved(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue) {+ t.Helper()+ deadline := time.Now().Add(webhookWaitTimeout)+ for {+ resp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{+ Name: issue.Name,+ }))+ require.NoError(t, err)+ if resp.Msg.ApprovalStatus == v1pb.Issue_APPROVED {+ return+ }+ if time.Now().After(deadline) {+ t.Fatalf("issue %s did not reach APPROVED within %s; current status %s",+ issue.Name, webhookWaitTimeout, resp.Msg.ApprovalStatus)+ }+ time.Sleep(200 * time.Millisecond)+ }+}++// waitForIssuePending blocks until the issue's approval-finding pipeline+// finishes and the issue is in PENDING — i.e., the rule has been resolved and+// the system is ready for an approver action. Fails the test on any other+// terminal status (APPROVED, REJECTED), which would indicate a setup bug.+func waitForIssuePending(ctx context.Context, t *testing.T, ctl *controller, issue *v1pb.Issue) {+ t.Helper()+ deadline := time.Now().Add(webhookWaitTimeout)+ for {+ resp, err := ctl.issueServiceClient.GetIssue(ctx, connect.NewRequest(&v1pb.GetIssueRequest{+ Name: issue.Name,+ }))+ require.NoError(t, err)+ switch resp.Msg.ApprovalStatus {+ case v1pb.Issue_PENDING:+ return+ case v1pb.Issue_CHECKING:+ // keep waiting+ default:+ t.Fatalf("issue %s reached unexpected approval status %s while waiting for PENDING",+ issue.Name, resp.Msg.ApprovalStatus)+ }+ if time.Now().After(deadline) {+ t.Fatalf("issue %s still CHECKING after %s", issue.Name, webhookWaitTimeout)+ }+ time.Sleep(500 * time.Millisecond)+ }+}diff --git a/backend/tests/webhook_test.go b/backend/tests/webhook_test.goindex 3ab4cd7077..2f58a1fbcd 100644--- a/backend/tests/webhook_test.go+++ b/backend/tests/webhook_test.go@@ -7,6 +7,7 @@ import ("io""net/http""net/http/httptest"+ "path""strings""sync""testing"@@ -146,7 +147,15 @@ func TestWebhookIntegration(t *testing.T) {require.NoError(t, err)defer ctl.Close(ctx)- // Create a test webhook server+ // Test webhook server.+ //+ // Body MUST be the literal "ok" — the Slack plugin's postMessage+ // (backend/plugin/webhook/slack/slack.go:247-249) treats any other body as+ // a delivery failure and triggers common.Retry (up to 3 attempts, 5s+ // exponential backoff). Without an "ok" response, every webhook event+ // would be delivered 1-3 times depending on test timing, causing+ // nondeterministic counts in the requireWebhookCount assertions across+ // the trigger matrix.collector := &webhookCollector{}webhookServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if err := collector.addRequest(r); err != nil {@@ -154,7 +163,7 @@ func TestWebhookIntegration(t *testing.T) {return}w.WriteHeader(http.StatusOK)- _, _ = w.Write([]byte(`{"success": true}`))+ _, _ = w.Write([]byte("ok"))}))defer webhookServer.Close()@@ -176,91 +185,525 @@ func TestWebhookIntegration(t *testing.T) {require.NoError(t, err)instance := instanceResp.Msg- // Create webhooks for issue events- for _, eventType := range []v1pb.Activity_Type{- v1pb.Activity_ISSUE_CREATED,- } {- _, err := ctl.projectServiceClient.AddWebhook(ctx, connect.NewRequest(&v1pb.AddWebhookRequest{- Project: ctl.project.Name,- Webhook: &v1pb.Webhook{- Type: v1pb.WebhookType_SLACK,- Title: fmt.Sprintf("Test Webhook for %s", eventType),- Url: webhookServer.URL,- NotificationTypes: []v1pb.Activity_Type{eventType},- },- }))- require.NoError(t, err)- }-t.Run("IssueWithPlanWebhookPayload", func(t *testing.T) {- // Reset webhook collector for this testcollector.reset()- // Create a plan with title and description- planTitle := "Database Migration Plan"+ project := ctl.createTestProject(ctx, t, "byt9398-i1")+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})+planDesc := "This plan creates a new database with important schema changes"planResp, err := ctl.planServiceClient.CreatePlan(ctx, connect.NewRequest(&v1pb.CreatePlanRequest{- Parent: ctl.project.Name,+ Parent: project.Name,Plan: &v1pb.Plan{- Name: planTitle,+ Name: "Database Migration Plan",Description: planDesc,- Specs: []*v1pb.Plan_Spec{- {- Id: uuid.NewString(),- Config: &v1pb.Plan_Spec_CreateDatabaseConfig{- CreateDatabaseConfig: &v1pb.Plan_CreateDatabaseConfig{- Target: instance.Name,- Database: "testdb",- },+ Specs: []*v1pb.Plan_Spec{{+ Id: uuid.NewString(),+ Config: &v1pb.Plan_Spec_CreateDatabaseConfig{+ CreateDatabaseConfig: &v1pb.Plan_CreateDatabaseConfig{+ Target: instance.Name,+ Database: "testdb",},},- },+ }},},}))require.NoError(t, err)- // Create an issue for webhook testing- issueResp, err := ctl.issueServiceClient.CreateIssue(ctx, connect.NewRequest(&v1pb.CreateIssueRequest{- Parent: ctl.project.Name,+ _, err = ctl.issueServiceClient.CreateIssue(ctx, connect.NewRequest(&v1pb.CreateIssueRequest{+ Parent: project.Name,Issue: &v1pb.Issue{Title: "Test webhook issue",- Description: "", // Empty description is OK+ Description: "",Type: v1pb.Issue_DATABASE_CHANGE,Plan: planResp.Msg.Name,},}))require.NoError(t, err)- require.NotEmpty(t, issueResp.Msg.Name)-- // Wait for webhook to be processed- time.Sleep(5 * time.Second)- // Verify webhook was triggered- requests := collector.getRequests()- require.GreaterOrEqual(t, len(requests), 1, "Expected at least 1 webhook")+ // Deterministic wait — replaces the original time.Sleep(5*time.Second).+ waitForWebhookCount(t, collector, project.Name, "Issue created", 1)- // Find and verify the issue creation webhook. Slack incoming webhook- // payloads intentionally omit top-level text to avoid rendering a- // duplicate message above the attachment card.+ // Find the captured request and verify payload contents.var foundCorrectWebhook bool- for _, req := range requests {+ for _, req := range collector.getRequests() {require.Equal(t, "POST", req.Method)require.Contains(t, req.Headers.Get("Content-Type"), "application/json")- title, desc, err := parseSlackWebhook(req.Body)+ _, desc, err := parseSlackWebhook(req.Body)require.NoError(t, err)-- // The webhook should use plan's description (title support is incomplete)- if desc == planDesc {- foundCorrectWebhook = true- t.Logf("✓ Webhook uses plan's description: %q", desc)- if title == "" {- t.Logf("⚠️ Issue title is empty (expected: %q)", planTitle)- }- break+ // The webhook payload uses the plan's description (issue-title support is incomplete).+ if desc != planDesc {+ continue}+ foundCorrectWebhook = true++ // I1 assertions: event identity and payload contents.+ require.True(t, matchesEvent(req, project.Name, "Issue created"),+ "first webhook section should contain 'Issue created'; got %q",+ firstSlackSectionText(req.Body))++ body := string(req.Body)+ require.Contains(t, body, project.Name, "payload should reference the project resource name")+ require.Contains(t, body, "Test webhook issue", "payload should contain the issue title")+ require.Contains(t, body, fmt.Sprintf("/projects/%s/issues/", path.Base(project.Name)),+ "payload should link to the project's issue resource")+ break}+ require.True(t, foundCorrectWebhook, "expected a webhook whose Slack description matches the plan description")+ })++ t.Run("PipelineCompletedAfterSkippingFailedTask", func(t *testing.T) {+ collector.reset()++ project := ctl.createTestProject(ctx, t, "byt9398-c4")++ // Create databases before registering the webhook so the implicit+ // PIPELINE_COMPLETED events fired by createDatabase (which internally+ // runs a createDatabaseConfig plan) do not pollute the collector.+ err := ctl.createDatabase(ctx, project, instance, nil, "byt9398_c4_pass", "")+ require.NoError(t, err)+ err = ctl.createDatabase(ctx, project, instance, nil, "byt9398_c4_fail", "")+ require.NoError(t, err)++ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{+ v1pb.Activity_PIPELINE_FAILED,+ v1pb.Activity_PIPELINE_COMPLETED,+ })++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c4_pass")},+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c4_fail")},+ })+ rollout := runAllTasks(ctx, t, ctl, plan)++ // Phase 1: failing task → exactly one PIPELINE_FAILED, no PIPELINE_COMPLETED.+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)+ requireWebhookCount(t, collector, project.Name, "Rollout completed", 0)++ // Phase 2: skip the failed task → PIPELINE_COMPLETED fires (the fix).+ skipFailedTasks(ctx, t, ctl, rollout)+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ })++ t.Run("PipelineCompleted_AllTasksDone", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-c1")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c1_a", ""))+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c1_b", ""))+ collector.reset() // flush any PIPELINE_COMPLETED from database creation+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,+ })++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c1_a")},+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c1_b")},+ })+ runAllTasks(ctx, t, ctl, plan)++ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)+ })++ t.Run("PipelineCompleted_DoneAndSkipped", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-c2")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c2_a", ""))+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c2_b", ""))+ collector.reset() // flush any PIPELINE_COMPLETED from database creation+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,+ })++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c2_a")},+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c2_b")},+ })+ rollout := createRolloutOnly(ctx, t, ctl, plan)+ skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c2_b"))+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c2_a"))++ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)+ })++ t.Run("PipelineCompleted_AllSkipped", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-c5")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c5_a", ""))+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c5_b", ""))+ collector.reset() // flush any PIPELINE_COMPLETED from database creation+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,+ })++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c5_a")},+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c5_b")},+ })+ rollout := createRolloutOnly(ctx, t, ctl, plan)+ skipAllTasks(ctx, t, ctl, rollout)++ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 0)+ })++ t.Run("PipelineCompleted_DoneAndFailedThenRetriedDone", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-c3")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c3_pass", ""))+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c3_fail", ""))+ collector.reset() // flush any PIPELINE_COMPLETED from database creation+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,+ })++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c3_pass")},+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c3_fail")},+ })+ rollout := runAllTasks(ctx, t, ctl, plan)++ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)+ requireWebhookCount(t, collector, project.Name, "Rollout completed", 0)++ unblockFailingTask(t, instanceDir, "byt9398_c3_fail")+ retryFailedTasks(ctx, t, ctl, rollout)++ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ })++ t.Run("PipelineCompleted_AllFailedThenAllSkipped", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-c6")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c6_a", ""))+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_c6_b", ""))+ collector.reset() // flush any PIPELINE_COMPLETED from database creation+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,+ })++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c6_a")},+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c6_b")},+ })+ rollout := runAllTasks(ctx, t, ctl, plan)++ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)++ skipAllTasks(ctx, t, ctl, rollout)+ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ })++ t.Run("PipelineCompleted_MixedRecovery", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-c7")+ for _, n := range []string{"byt9398_c7_done", "byt9398_c7_skip", "byt9398_c7_retry", "byt9398_c7_skipfailed"} {+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, n, ""))+ }+ collector.reset() // flush any PIPELINE_COMPLETED from database creation+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{+ v1pb.Activity_PIPELINE_FAILED, v1pb.Activity_PIPELINE_COMPLETED,+ })++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_done")},+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_skip")},+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_retry")},+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_c7_skipfailed")},+ })++ rollout := createRolloutOnly(ctx, t, ctl, plan)+ skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skip"))+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_done"))+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"))+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skipfailed"))++ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 60*time.Second)+ requireWebhookCount(t, collector, project.Name, "Rollout completed", 0)++ // Unblock dbRetry only — dbSkipFailed's __force_fail_target table remains+ // absent in its own .db file, so its retry would still fail.+ unblockFailingTask(t, instanceDir, "byt9398_c7_retry")+ runTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"))+ waitForTaskStatus(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_retry"), v1pb.Task_DONE, 30*time.Second)+ skipTaskByDB(ctx, t, ctl, rollout, dbTargetName(instance, "byt9398_c7_skipfailed"))++ waitForWebhookCount(t, collector, project.Name, "Rollout completed", 1)+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ })++ t.Run("PipelineFailed_SingleTaskFails", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-f1")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f1_fail", ""))+ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f1_fail")},+ })+ runAllTasks(ctx, t, ctl, plan)+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ })++ t.Run("PipelineFailed_DedupOnSecondTaskFailure", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-f2")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f2_a", ""))+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f2_b", ""))+ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f2_a")},+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f2_b")},+ })+ rollout := runAllTasks(ctx, t, ctl, plan)- require.True(t, foundCorrectWebhook, "Webhook should use plan's description")+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)++ // Both tasks have failed. ClaimPipelineFailureNotification's PK collision+ // must dedupe — assert exactly 1 even after both terminal.+ requireWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ })++ t.Run("PipelineFailed_RetryFailsAgain", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-f3")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_f3_fail", ""))+ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_PIPELINE_FAILED})+ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedFailingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_f3_fail")},+ })+ rollout := runAllTasks(ctx, t, ctl, plan)++ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 1)+ waitForAllTasksTerminal(ctx, t, ctl, rollout, 30*time.Second)++ // BatchRunTasks resets the dedup row before enqueuing the retry, so the+ // second failure must re-fire PIPELINE_FAILED. We deliberately do NOT call+ // unblockFailingTask — the retry should fail again.+ retryFailedTasks(ctx, t, ctl, rollout)+ waitForWebhookCount(t, collector, project.Name, "Rollout failed", 2)+ })++ t.Run("IssueCreated_NoLeakageFromOtherEvents", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-i2")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_i2_db", ""))++ disableSelfApproval(ctx, t, ctl, project)+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})+ appr := provisionApprover(ctx, t, ctl, project, "i2", "roles/projectOwner")++ collector.reset() // flush any creation-time webhook events+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_CREATED})++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_i2_db")},+ })+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "I2 issue")++ waitForWebhookCount(t, collector, project.Name, "Issue created", 1)+ waitForIssuePending(ctx, t, ctl, issue)++ // Drive an approval. The webhook is NOT subscribed to ISSUE_APPROVED, so the+ // count must stay at 1. Wait for the issue to actually reach APPROVED before+ // asserting — that proves the approval pipeline ran to completion and any+ // approval-related webhooks have already had their chance to fire.+ approveIssueAs(ctx, t, ctl, issue, appr)+ waitForIssueApproved(ctx, t, ctl, issue)+ requireWebhookCount(t, collector, project.Name, "Issue created", 1)+ requireWebhookCount(t, collector, project.Name, "Issue approved", 0)+ })++ t.Run("IssueApprovalRequested_FiresWhenRequired", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-a1")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_a1_db", ""))++ disableSelfApproval(ctx, t, ctl, project)+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})+ _ = provisionApprover(ctx, t, ctl, project, "a1", "roles/projectOwner")++ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVAL_REQUESTED})++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_a1_db")},+ })+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "A1 issue")+ waitForIssuePending(ctx, t, ctl, issue)++ waitForWebhookCount(t, collector, project.Name, "Approval required", 1)+ })++ t.Run("IssueApprovalRequested_NotFiredWhenUnused", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-a2")+ // The workspace has a default catch-all approval rule. Clear it so that+ // no rule applies to A2's project, then assert no ISSUE_APPROVAL_REQUESTED fires.+ clearWorkspaceApprovalRules(ctx, t, ctl)+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_a2_db", ""))++ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVAL_REQUESTED})++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_a2_db")},+ })+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "A2 issue")++ // Wait for the approval-finding pipeline to finish (any terminal status —+ // with no rule applying, the issue auto-resolves). After this wait the+ // runner has had its chance to fire any approval-related webhook, so the+ // absence assertion is no longer racing the runner.+ waitForApprovalFindingDone(ctx, t, ctl, issue)+ requireWebhookCount(t, collector, project.Name, "Approval required", 0)+ })++ t.Run("IssueApproved_SingleStep", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-ap1")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_ap1_db", ""))++ clearWorkspaceApprovalRules(ctx, t, ctl)+ disableSelfApproval(ctx, t, ctl, project)+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})+ appr := provisionApprover(ctx, t, ctl, project, "ap1", "roles/projectOwner")++ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVED})++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_ap1_db")},+ })+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "AP1 issue")+ waitForIssuePending(ctx, t, ctl, issue)++ approveIssueAs(ctx, t, ctl, issue, appr)+ waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)+ })++ t.Run("IssueApproved_MultiStepOnlyFiresAtFinal", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-ap2")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_ap2_db", ""))++ clearWorkspaceApprovalRules(ctx, t, ctl)+ disableSelfApproval(ctx, t, ctl, project)+ installWorkspaceApprovalRule(ctx, t, ctl, project,+ []string{"roles/projectDeveloper", "roles/projectOwner"})+ apprStep1 := provisionApprover(ctx, t, ctl, project, "ap2-step1", "roles/projectDeveloper")+ apprStep2 := provisionApprover(ctx, t, ctl, project, "ap2-step2", "roles/projectOwner")++ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_APPROVED})++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_ap2_db")},+ })+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "AP2 issue")+ waitForIssuePending(ctx, t, ctl, issue)++ // After step-1 approval the issue must remain PENDING (step 2 still required).+ // waitForIssuePending is a stable post-condition — by the time it returns the+ // approval runner has updated the next-step approver state and the gRPC+ // handler has already decided not to call NotifyIssueApproved (because+ // approved == false). No goroutine for ISSUE_APPROVED was ever spawned, so+ // the count assertion below is race-free.+ approveIssueAs(ctx, t, ctl, issue, apprStep1)+ waitForIssuePending(ctx, t, ctl, issue)+ requireWebhookCount(t, collector, project.Name, "Issue approved", 0)++ approveIssueAs(ctx, t, ctl, issue, apprStep2)+ waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)+ })++ t.Run("IssueSentBack_FiresOnRejection", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-sb1")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_sb1_db", ""))++ clearWorkspaceApprovalRules(ctx, t, ctl)+ disableSelfApproval(ctx, t, ctl, project)+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})+ appr := provisionApprover(ctx, t, ctl, project, "sb1", "roles/projectOwner")++ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{v1pb.Activity_ISSUE_SENT_BACK})++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_sb1_db")},+ })+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "SB1 issue")+ waitForIssuePending(ctx, t, ctl, issue)++ rejectIssueAs(ctx, t, ctl, issue, appr, "needs more context")+ waitForWebhookCount(t, collector, project.Name, "Issue sent back", 1)+ })++ t.Run("IssueSentBack_ThenReapproved_E2E", func(t *testing.T) {+ collector.reset()+ project := ctl.createTestProject(ctx, t, "byt9398-sb2")+ require.NoError(t, ctl.createDatabase(ctx, project, instance, nil, "byt9398_sb2_db", ""))++ clearWorkspaceApprovalRules(ctx, t, ctl)+ disableSelfApproval(ctx, t, ctl, project)+ installWorkspaceApprovalRule(ctx, t, ctl, project, []string{"roles/projectOwner"})+ appr := provisionApprover(ctx, t, ctl, project, "sb2", "roles/projectOwner")++ collector.reset()+ addWebhookForEvents(ctx, t, ctl, project, webhookServer.URL, []v1pb.Activity_Type{+ v1pb.Activity_ISSUE_SENT_BACK,+ v1pb.Activity_ISSUE_APPROVED,+ v1pb.Activity_ISSUE_APPROVAL_REQUESTED,+ })++ plan := createPlanWithSpecs(ctx, t, ctl, project, []taskSpec{+ {seedPassingSheet(ctx, t, ctl, project), dbTargetName(instance, "byt9398_sb2_db")},+ })+ issue := createIssueForPlan(ctx, t, ctl, project, plan, "SB2 issue")+ waitForIssuePending(ctx, t, ctl, issue)++ rejectIssueAs(ctx, t, ctl, issue, appr, "fix and resubmit")+ waitForWebhookCount(t, collector, project.Name, "Issue sent back", 1)++ // Issue creator (default ctl token) re-requests approval. Note: RequestIssue+ // does NOT reset ApprovalFindingDone, so waitForIssuePending would return+ // immediately. Wait for the second "Approval required" webhook instead —+ // RequestIssue calls approval.NotifyApprovalRequested directly.+ requestIssueAsCreator(ctx, t, ctl, issue, "addressed feedback")+ waitForWebhookCount(t, collector, project.Name, "Approval required", 2)++ approveIssueAs(ctx, t, ctl, issue, appr)+ waitForWebhookCount(t, collector, project.Name, "Issue approved", 1)+ })++ // Regression for the second Codex P1 finding on PR #20212. Empty BatchSkipTasks+ // must be rejected at the API boundary, otherwise it sneaks past the+ // task-parse, environmentSet permission, and store mutation paths and would+ // run ResetPlanWebhookDelivery on a request that did nothing — re-arming the+ // dedup ledger for any plan the caller can name.+ t.Run("BatchSkipTasks_RejectsEmptyTasks", func(t *testing.T) {+ project := ctl.createTestProject(ctx, t, "byt9398-empty-skip")+ // Parent format doesn't matter here; the empty-tasks guard fires first.+ _, err := ctl.rolloutServiceClient.BatchSkipTasks(ctx, connect.NewRequest(&v1pb.BatchSkipTasksRequest{+ Parent: project.Name + "/plans/-/rollouts/-/stages/-",+ Tasks: []string{},+ }))+ require.Error(t, err)+ require.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err),+ "BatchSkipTasks must reject empty Tasks at the API boundary")})}