Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: HA-compatible webhook delivery for pipeline events (#18708)
Danny Xu6f7441d9a0df
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.goindex 9880d9fa9f..809b82b919 100644--- a/backend/api/v1/rollout_service.go+++ b/backend/api/v1/rollout_service.go@@ -4,6 +4,7 @@ import ("context""database/sql""fmt"+ "log/slog""slices""strings""time"@@ -15,6 +16,7 @@ import ("google.golang.org/protobuf/types/known/timestamppb""github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/dbfactory""github.com/bytebase/bytebase/backend/component/iam""github.com/bytebase/bytebase/backend/component/state"@@ -654,6 +656,12 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Requestreturn nil, connect.NewError(connect.CodeNotFound, errors.Errorf("rollout (plan) %v not found", planID))}+ // Reset notification state so user gets fresh feedback on retry+ if err := s.store.ResetPlanWebhookDelivery(ctx, planID); err != nil {+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))+ // Don't fail the request - notification is non-critical+ }+issueN, err := s.store.GetIssue(ctx, &store.FindIssueMessage{ProjectID: &projectID,PlanUID: &planID,@@ -802,7 +810,6 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Requesreturn nil, connect.NewError(connect.CodeInternal, errors.New("user not found"))}var taskUIDs []int- var tasksToSkip []*store.TaskMessageenvironmentSet := map[string]struct{}{}for _, task := range request.Tasks {_, _, _, taskID, err := common.GetProjectIDPlanIDStageIDTaskID(task)@@ -814,7 +821,6 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Requesreturn nil, connect.NewError(connect.CodeNotFound, errors.Errorf("task %v not found in the rollout", taskID))}taskUIDs = append(taskUIDs, taskID)- tasksToSkip = append(tasksToSkip, taskMsg)environmentSet[taskMsg.Environment] = struct{}{}}@@ -832,9 +838,8 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Requesreturn nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to skip tasks"))}- for _, task := range tasksToSkip {- s.stateCfg.TaskSkippedOrDoneChan <- task.ID- }+ // Signal to check if plan is complete and successful (may send PIPELINE_COMPLETED)+ s.stateCfg.PlanCompletionCheckChan <- planIDreturn connect.NewResponse(&v1pb.BatchSkipTasksResponse{}), nil}@@ -942,6 +947,8 @@ func (s *RolloutService) BatchCancelTaskRuns(ctx context.Context, req *connect.Rreturn nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to batch patch task run status to canceled"))}+ // Note: Don't signal plan completion - canceling is interrupting execution, not completing the plan+return connect.NewResponse(&v1pb.BatchCancelTaskRunsResponse{}), nil}diff --git a/backend/component/state/state.go b/backend/component/state/state.goindex 931fe1b573..a01174b5aa 100644--- a/backend/component/state/state.go+++ b/backend/component/state/state.go@@ -20,9 +20,6 @@ type State struct {// RunningPlanCheckRunsCancelFunc is the cancelFunc of running plan checks.RunningPlanCheckRunsCancelFunc sync.Map // map[planCheckRunUID]context.CancelFunc- // TaskSkippedOrDoneChan is the channel for notifying the task is skipped or done.- TaskSkippedOrDoneChan chan int-// PlanCheckTickleChan is the tickler for plan check scheduler.PlanCheckTickleChan chan int// TaskRunTickleChan is the tickler for task run scheduler.@@ -30,13 +27,16 @@ type State struct {// RolloutCreationChan is the channel for automatic rollout creation.RolloutCreationChan chan int64++ // PlanCompletionCheckChan signals when a plan might be complete (for PIPELINE_COMPLETED webhook).+ PlanCompletionCheckChan chan int64}func New() (*State, error) {return &State{- TaskSkippedOrDoneChan: make(chan int, 1000),- PlanCheckTickleChan: make(chan int, 1000),- TaskRunTickleChan: make(chan int, 1000),- RolloutCreationChan: make(chan int64, 100),+ PlanCheckTickleChan: make(chan int, 1000),+ TaskRunTickleChan: make(chan int, 1000),+ RolloutCreationChan: make(chan int64, 100),+ PlanCompletionCheckChan: make(chan int64, 1000),}, nil}diff --git a/backend/component/webhook/event.go b/backend/component/webhook/event.goindex 2967a31c2a..5cc06fc321 100644--- a/backend/component/webhook/event.go+++ b/backend/component/webhook/event.go@@ -1,8 +1,6 @@package webhookimport (- "time"-storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store")@@ -87,17 +85,7 @@ type EventIssueSentBack struct {}type EventRolloutFailed struct {- Rollout *Rollout- FailedTasks []FailedTask-}--type FailedTask struct {- TaskID int64- TaskName string- DatabaseName string- InstanceName string- ErrorMessage string- FailedAt time.Time+ Rollout *Rollout}type EventRolloutCompleted struct {diff --git a/backend/component/webhook/manager.go b/backend/component/webhook/manager.goindex e206f2f438..2dd03fa4f8 100644--- a/backend/component/webhook/manager.go+++ b/backend/component/webhook/manager.go@@ -133,18 +133,7 @@ func (m *Manager) getWebhookContextFromEvent(ctx context.Context, e *Event, evenif e.RolloutFailed != nil {rollout = e.RolloutFailed.Rolloutlink = fmt.Sprintf("%s/projects/%s/plans/%d/rollout", externalURL, e.Project.ResourceID, rollout.UID)- failedTasks := make([]webhook.FailedTaskInfo, 0, len(e.RolloutFailed.FailedTasks))- for _, task := range e.RolloutFailed.FailedTasks {- failedTasks = append(failedTasks, webhook.FailedTaskInfo{- Name: task.TaskName,- Instance: task.InstanceName,- Database: task.DatabaseName,- ErrorMessage: task.ErrorMessage,- FailedAt: task.FailedAt.Format(time.RFC3339),- })- }- webhookCtx.FailedTasks = failedTasks- webhookCtx.Description = fmt.Sprintf("%d task(s) failed", len(failedTasks))+ webhookCtx.Description = "Rollout failed"}case storepb.Activity_PIPELINE_COMPLETED:diff --git a/backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql b/backend/migrator/migration/3.14/0016##plan_webhook_delivery.sqlnew file mode 100644index 0000000000..94b7a5cc85--- /dev/null+++ b/backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql@@ -0,0 +1,9 @@+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).+-- One row per plan at any time - mutually exclusive events.+-- Row is deleted when user clicks BatchRunTasks to reset notification state.+CREATE TABLE plan_webhook_delivery (+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'+ event_type TEXT NOT NULL,+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()+);diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sqlindex 905d6a99ba..fe59f513e4 100644--- a/backend/migrator/migration/LATEST.sql+++ b/backend/migrator/migration/LATEST.sql@@ -229,6 +229,16 @@ CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHERALTER SEQUENCE plan_check_run_id_seq RESTART WITH 101;+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).+-- One row per plan at any time - mutually exclusive events.+-- Row is deleted when user clicks BatchRunTasks to reset notification state.+CREATE TABLE plan_webhook_delivery (+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'+ event_type TEXT NOT NULL,+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()+);+-- task table stores the task for a planCREATE TABLE task (id serial PRIMARY KEY,diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex 4c70b914bb..d4e7e45ca1 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -12,8 +12,8 @@ import (func TestLatestVersion(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)- require.Equal(t, semver.MustParse("3.14.15"), *files[len(files)-1].version)- require.Equal(t, "migration/3.14/0015##task_run_available_status.sql", files[len(files)-1].path)+ require.Equal(t, semver.MustParse("3.14.16"), *files[len(files)-1].version)+ require.Equal(t, "migration/3.14/0016##plan_webhook_delivery.sql", files[len(files)-1].path)}func TestVersionUnique(t *testing.T) {diff --git a/backend/runner/taskrun/pipeline_events.go b/backend/runner/taskrun/pipeline_events.godeleted file mode 100644index 25e4b697dd..0000000000--- a/backend/runner/taskrun/pipeline_events.go+++ /dev/null@@ -1,78 +0,0 @@-package taskrun--import (- "sync"- "time"-- "github.com/bytebase/bytebase/backend/component/webhook"-)--// PipelineFailureWindow tracks failed tasks for aggregation.-type PipelineFailureWindow struct {- firstFailureTime time.Time- failedTasks []webhook.FailedTask- notificationSent bool- timer *time.Timer-}--// PipelineEventsTracker manages failure aggregation windows per plan.-type PipelineEventsTracker struct {- mu sync.RWMutex- windows map[int64]*PipelineFailureWindow // planID -> window-}--// NewPipelineEventsTracker creates a new pipeline events tracker.-func NewPipelineEventsTracker() *PipelineEventsTracker {- return &PipelineEventsTracker{- windows: make(map[int64]*PipelineFailureWindow),- }-}--// RecordTaskFailure adds a failed task to the aggregation window.-func (t *PipelineEventsTracker) RecordTaskFailure(planID int64, task webhook.FailedTask, onAggregated func([]webhook.FailedTask)) {- t.mu.Lock()- defer t.mu.Unlock()-- window, exists := t.windows[planID]- if !exists || window.notificationSent {- // Start new window- window = &PipelineFailureWindow{- firstFailureTime: time.Now(),- failedTasks: []webhook.FailedTask{task},- notificationSent: false,- }- t.windows[planID] = window-- // Set 5-minute timer- window.timer = time.AfterFunc(5*time.Minute, func() {- t.mu.Lock()- defer t.mu.Unlock()-- if w, ok := t.windows[planID]; ok && !w.notificationSent {- w.notificationSent = true- // Copy tasks to avoid holding lock during callback- tasksCopy := make([]webhook.FailedTask, len(w.failedTasks))- copy(tasksCopy, w.failedTasks)- // Delete window after sending to prevent memory leak- delete(t.windows, planID)- onAggregated(tasksCopy)- }- })- } else {- // Add to existing window- window.failedTasks = append(window.failedTasks, task)- }-}--// Clear removes the window for a plan (call after pipeline completes).-func (t *PipelineEventsTracker) Clear(planID int64) {- t.mu.Lock()- defer t.mu.Unlock()-- if window, exists := t.windows[planID]; exists {- if window.timer != nil {- window.timer.Stop()- }- delete(t.windows, planID)- }-}diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.goindex 3f5d0de37e..d4b8a8271c 100644--- a/backend/runner/taskrun/running_scheduler.go+++ b/backend/runner/taskrun/running_scheduler.go@@ -199,7 +199,32 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRunUID int, task *streturn}s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_FAILED, taskRunResult.Detail)- s.recordPipelineFailure(ctx, task, taskRunResult.Detail)++ // Immediately try to send PIPELINE_FAILED webhook (HA-safe atomic claim)+ claimed, err := s.store.ClaimPipelineFailureNotification(ctx, task.PlanID)+ if err != nil {+ slog.Error("failed to claim pipeline failure notification", log.BBError(err))+ } else if claimed {+ // Get plan and project for webhook+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})+ if err != nil || plan == nil {+ slog.Error("failed to get plan for failure webhook", log.BBError(err))+ } else {+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})+ if err != nil || project == nil {+ slog.Error("failed to get project for failure webhook", log.BBError(err))+ } else {+ // Send PIPELINE_FAILED webhook+ s.webhookManager.CreateEvent(ctx, &webhook.Event{+ Type: storepb.Activity_PIPELINE_FAILED,+ Project: webhook.NewProject(project),+ RolloutFailed: &webhook.EventRolloutFailed{+ Rollout: webhook.NewRollout(plan),+ },+ })+ }+ }+ }returncase done && err == nil:@@ -229,7 +254,9 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRunUID int, task *streturn}s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_DONE, "")- s.stateCfg.TaskSkippedOrDoneChan <- task.ID++ // Signal to check if plan is complete and successful (may send PIPELINE_COMPLETED)+ s.stateCfg.PlanCompletionCheckChan <- task.PlanIDreturndefault:// This case should not happen in normal flow, but adding for completeness@@ -247,63 +274,6 @@ func (*Scheduler) createActivityForTaskRunStatusUpdate(_ context.Context, _ *sto// No webhook events for task run status updates}-func (s *Scheduler) recordPipelineFailure(ctx context.Context, task *store.TaskMessage, errDetail string) {- if err := func() error {- plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})- if err != nil {- return errors.Wrapf(err, "failed to get plan")- }- if plan == nil {- return errors.Errorf("plan %v not found", task.PlanID)- }-- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})- if err != nil {- return errors.Wrapf(err, "failed to get project")- }- if project == nil {- return errors.Errorf("project %v not found", plan.ProjectID)- }-- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})- if err != nil {- return errors.Wrapf(err, "failed to get instance")- }-- failedTask := webhook.FailedTask{- TaskID: int64(task.ID),- TaskName: task.Type.String(),- DatabaseName: task.GetDatabaseName(),- InstanceName: instance.Metadata.Title,- ErrorMessage: errDetail,- FailedAt: time.Now(),- }-- s.pipelineEvents.RecordTaskFailure(- plan.UID,- failedTask,- func(failedTasks []webhook.FailedTask) {- // Use background context to avoid cancellation issues- webhookCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)- defer cancel()-- s.webhookManager.CreateEvent(webhookCtx, &webhook.Event{- Type: storepb.Activity_PIPELINE_FAILED,- Project: webhook.NewProject(project),- RolloutFailed: &webhook.EventRolloutFailed{- Rollout: webhook.NewRollout(plan),- FailedTasks: failedTasks,- },- })- },- )-- return nil- }(); err != nil {- slog.Error("failed to record pipeline failure", log.BBError(err))- }-}-// isSequentialTask returns whether the task should be executed sequentially.func isSequentialTask(task *store.TaskMessage) bool {//exhaustive:enforcediff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.goindex 90d4049f1a..d184262f6d 100644--- a/backend/runner/taskrun/scheduler.go+++ b/backend/runner/taskrun/scheduler.go@@ -27,7 +27,6 @@ type Scheduler struct {webhookManager *webhook.ManagerexecutorMap map[storepb.Task_Type]Executorprofile *config.Profile- pipelineEvents *PipelineEventsTracker}// NewScheduler will create a new scheduler.@@ -43,7 +42,6 @@ func NewScheduler(webhookManager: webhookManager,profile: profile,executorMap: map[storepb.Task_Type]Executor{},- pipelineEvents: NewPipelineEventsTracker(),}}@@ -85,30 +83,21 @@ func (s *Scheduler) runTaskCompletionListener(ctx context.Context) {slog.Error("runTaskCompletionListener PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))}}()- slog.Info("Task completion listener started")+ slog.Info("Plan completion checker started")for {select {- case taskUID := <-s.stateCfg.TaskSkippedOrDoneChan:- if err := func() error {- task, err := s.store.GetTaskByID(ctx, taskUID)- if err != nil {- return errors.Wrapf(err, "failed to get task")- }-- // Check if entire plan is complete and handle webhooks- s.checkPlanCompletion(ctx, task.PlanID)-- return nil- }(); err != nil {- slog.Error("failed to handle task completion", log.BBError(err))- }case <-ctx.Done():return+ case planID := <-s.stateCfg.PlanCompletionCheckChan:+ s.checkPlanCompletion(ctx, planID)}}}+// checkPlanCompletion checks if all tasks in a plan are complete and successful.+// If so, sends PIPELINE_COMPLETED webhook.+// Called when tasks are marked DONE/SKIPPED, or when tasks are skipped/canceled via API.func (s *Scheduler) checkPlanCompletion(ctx context.Context, planID int64) {// Get all tasks for this plantasks, err := s.store.ListTasks(ctx, &store.TaskFind{PlanID: &planID})@@ -117,87 +106,44 @@ func (s *Scheduler) checkPlanCompletion(ctx context.Context, planID int64) {return}- // Check if all tasks are in terminal state- allComplete := true- hasFailures := false- var earliestStart, latestEnd *time.Time-+ // Check if all tasks are complete (DONE or SKIPPED)for _, task := range tasks {status := task.LatestTaskRunStatus- // Check if task is in terminal state- isTerminal := status == storepb.TaskRun_DONE ||- status == storepb.TaskRun_FAILED ||- status == storepb.TaskRun_CANCELED ||+ // Only DONE and SKIPPED are considered complete+ // FAILED and CANCELED are not complete states+ isComplete := status == storepb.TaskRun_DONE ||status == storepb.TaskRun_SKIPPED ||task.Payload.GetSkipped()- if !isTerminal {- allComplete = false- break- }-- // Track failures (excluding skipped tasks)- if status == storepb.TaskRun_FAILED && !task.Payload.GetSkipped() {- hasFailures = true- }-- // Track start/end times for metrics- if task.RunAt != nil {- if earliestStart == nil || task.RunAt.Before(*earliestStart) {- earliestStart = task.RunAt- }- }- if task.UpdatedAt != nil {- if latestEnd == nil || task.UpdatedAt.After(*latestEnd) {- latestEnd = task.UpdatedAt- }+ if !isComplete {+ // Not all tasks complete - no webhook+ return}}- // Not all tasks complete yet- if !allComplete {+ // All tasks complete and successful - try to claim completion notification+ claimed, err := s.store.ClaimPipelineCompletionNotification(ctx, planID)+ if err != nil {+ slog.Error("failed to claim pipeline completion notification", log.BBError(err))return}-- // Always clear the failure window when plan completes- s.pipelineEvents.Clear(planID)-- // Only send completion webhook if there were no failures- if hasFailures {- return+ if !claimed {+ return // Already sent}- // Get plan, project and issue for webhook+ // Get plan and project for webhookplan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &planID})- if err != nil {+ if err != nil || plan == nil {slog.Error("failed to get plan for completion webhook", log.BBError(err))return}- if plan == nil {- slog.Error("plan not found for completion webhook", slog.Int64("plan_id", planID))- return- }project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})- if err != nil {+ if err != nil || project == nil {slog.Error("failed to get project for completion webhook", log.BBError(err))return}- if project == nil {- slog.Error("project not found for completion webhook", slog.String("project_id", plan.ProjectID))- return- }-- issueN, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &planID})- if err != nil {- slog.Error("failed to get issue for completion webhook", log.BBError(err))- return- }- if issueN == nil {- slog.Error("issue not found for completion webhook", slog.Int64("plan_id", planID))- return- }// Send PIPELINE_COMPLETED webhooks.webhookManager.CreateEvent(ctx, &webhook.Event{diff --git a/backend/store/plan_webhook_delivery.go b/backend/store/plan_webhook_delivery.gonew file mode 100644index 0000000000..30836e531b--- /dev/null+++ b/backend/store/plan_webhook_delivery.go@@ -0,0 +1,52 @@+package store++import (+ "context"+ "database/sql"+)++// ResetPlanWebhookDelivery deletes the webhook delivery record for a plan.+// Called when user clicks BatchRunTasks to enable new notifications on retry.+func (s *Store) ResetPlanWebhookDelivery(ctx context.Context, planID int64) error {+ query := `DELETE FROM plan_webhook_delivery WHERE plan_id = $1`+ _, err := s.GetDB().ExecContext(ctx, query, planID)+ return err+}++// ClaimPipelineFailureNotification attempts to claim the right to send PIPELINE_FAILED webhook.+// Returns true if claimed (should send), false if already sent or claimed by another replica.+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.+func (s *Store) ClaimPipelineFailureNotification(ctx context.Context, planID int64) (bool, error) {+ query := `+ INSERT INTO plan_webhook_delivery (plan_id, event_type)+ VALUES ($1, 'PIPELINE_FAILED')+ ON CONFLICT (plan_id) DO NOTHING+ RETURNING plan_id+ `++ var id int64+ err := s.GetDB().QueryRowContext(ctx, query, planID).Scan(&id)+ if err == sql.ErrNoRows {+ return false, nil // Already exists+ }+ return err == nil, err+}++// ClaimPipelineCompletionNotification attempts to claim the right to send PIPELINE_COMPLETED webhook.+// Returns true if claimed (should send), false if already sent or claimed by another replica.+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.+func (s *Store) ClaimPipelineCompletionNotification(ctx context.Context, planID int64) (bool, error) {+ query := `+ INSERT INTO plan_webhook_delivery (plan_id, event_type)+ VALUES ($1, 'PIPELINE_COMPLETED')+ ON CONFLICT (plan_id) DO NOTHING+ RETURNING plan_id+ `++ var id int64+ err := s.GetDB().QueryRowContext(ctx, query, planID).Scan(&id)+ if err == sql.ErrNoRows {+ return false, nil // Already exists+ }+ return err == nil, err+}diff --git a/docs/plans/2025-12-31-ha-webhook-delivery-design.md b/docs/plans/2025-12-31-ha-webhook-delivery-design.mdnew file mode 100644index 0000000000..f672c35f11--- /dev/null+++ b/docs/plans/2025-12-31-ha-webhook-delivery-design.md@@ -0,0 +1,395 @@+# HA-Compatible Webhook Delivery Design++**Date:** 2025-12-31+**Status:** Design+**Author:** Danny++## Overview++This design makes pipeline webhook notifications (PIPELINE_FAILED and PIPELINE_COMPLETED) work correctly in High Availability (HA) deployments where multiple Bytebase replicas run simultaneously. It replaces the in-memory `TaskSkippedOrDoneChan` channel with a database-backed idempotent delivery system.++## Problem++The current webhook notification system uses an in-memory channel (`state.TaskSkippedOrDoneChan`) which breaks in HA:++```go+// backend/component/state/state.go+type State struct {+ TaskSkippedOrDoneChan chan int // In-memory, not shared across replicas+}++// backend/runner/taskrun/running_scheduler.go+s.stateCfg.TaskSkippedOrDoneChan <- task.ID // Send notification++// backend/runner/taskrun/scheduler.go+case taskUID := <-s.stateCfg.TaskSkippedOrDoneChan: // Receive notification+ s.checkPlanCompletion(ctx, task.PlanID)+```++**HA Issues:**+- Replica A marks task as done, sends to its local channel+- Replica B receives user request, doesn't see anything in its channel+- Both replicas might process same task failure → duplicate webhooks+- Or neither replica sends webhook → missed notification++## Requirements++1. **No spam**: Send PIPELINE_FAILED only once when first task fails+2. **Retry support**: Send new PIPELINE_FAILED when user clicks "Retry Tasks"+3. **Completion**: Send PIPELINE_COMPLETED once when all tasks succeed+4. **HA-safe**: Multiple replicas shouldn't send duplicate webhooks+5. **Simple**: Avoid complex aggregation windows or polling++## Solution: Database-Backed Delivery Log++Use a single database table with PRIMARY KEY constraint to ensure exactly-once delivery across HA replicas.++### Key Insight++- `PIPELINE_FAILED` and `PIPELINE_COMPLETED` are **mutually exclusive** per plan+- User clicking "Retry Tasks" is an explicit action that resets notification state+- Database PRIMARY KEY constraint provides atomic claim mechanism++### Schema++```sql+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).+-- One row per plan at any time - mutually exclusive events.+-- Row is deleted when user clicks BatchRunTasks to reset notification state.+CREATE TABLE webhook_delivery_log (+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'+ event_type TEXT NOT NULL,+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()+);+```++**Design choices:**+- `plan_id` as PRIMARY KEY enforces uniqueness automatically+- Keep `event_type` for audit trail (debugging which event was sent)+- No CHECK constraint - application code controls values, comments document them+- No separate indexes needed - PRIMARY KEY provides all we need++## Architecture++### 1. When Task Run Fails++Location: `backend/runner/taskrun/running_scheduler.go`++```go+// Add after task run status is marked as FAILED+func (s *Scheduler) handleTaskRunFailure(ctx context.Context, taskRun *store.TaskRunMessage) {+ task, err := s.store.GetTaskByID(ctx, taskRun.TaskID)+ if err != nil {+ slog.Error("failed to get task for failure notification", log.BBError(err))+ return+ }++ // Try to claim notification (only one replica succeeds)+ claimed, err := s.store.ClaimPipelineFailureNotification(ctx, task.PlanID)+ if err != nil {+ slog.Error("failed to claim pipeline failure notification", log.BBError(err))+ return+ }+ if !claimed {+ // Already sent by this or another replica+ return+ }++ // Get all failed tasks for this plan+ failures, err := s.getFailedTaskRuns(ctx, task.PlanID)+ if err != nil {+ slog.Error("failed to get failed task runs", log.BBError(err))+ return+ }++ // Get plan/project context+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})+ if err != nil || plan == nil {+ slog.Error("failed to get plan for failure webhook", log.BBError(err))+ return+ }++ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})+ if err != nil || project == nil {+ slog.Error("failed to get project for failure webhook", log.BBError(err))+ return+ }++ // Send webhook+ s.webhookManager.CreateEvent(ctx, &webhook.Event{+ Type: storepb.Activity_PIPELINE_FAILED,+ Project: webhook.NewProject(project),+ PipelineFailed: &webhook.EventPipelineFailed{+ Rollout: webhook.NewRollout(plan),+ FailedTasks: failures,+ },+ })+}+```++### 2. When Pipeline Completes++Location: `backend/runner/taskrun/scheduler.go` (existing `checkPlanCompletion` function)++```go+func (s *Scheduler) checkPlanCompletion(ctx context.Context, planID int64) {+ // ... existing task completion check logic ...++ if !allComplete {+ return+ }++ if hasFailures {+ return // Don't send completion if there were failures+ }++ // Try to claim completion notification (HA-safe)+ claimed, err := s.store.ClaimPipelineCompletionNotification(ctx, planID)+ if err != nil {+ slog.Error("failed to claim pipeline completion notification", log.BBError(err))+ return+ }+ if !claimed {+ return // Already sent+ }++ // ... existing webhook send code ...+ s.webhookManager.CreateEvent(ctx, &webhook.Event{+ Type: storepb.Activity_PIPELINE_COMPLETED,+ Project: webhook.NewProject(project),+ RolloutCompleted: &webhook.EventRolloutCompleted{+ Rollout: webhook.NewRollout(plan),+ },+ })+}+```++### 3. When User Clicks "Retry Tasks"++Location: `backend/api/v1/rollout_service.go` (existing `BatchRunTasks` function)++```go+func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request[v1pb.BatchRunTasksRequest]) (*connect.Response[v1pb.BatchRunTasksResponse], error) {+ // ... existing validation code ...++ // Reset notification state so user gets fresh feedback on retry+ if err := s.store.ResetPipelineNotification(ctx, planID); err != nil {+ slog.Error("failed to reset pipeline notification", log.BBError(err))+ // Don't fail the request - notification is non-critical+ }++ // ... existing CreatePendingTaskRuns call ...++ return connect.NewResponse(&v1pb.BatchRunTasksResponse{}), nil+}+```++## Store Methods++Location: `backend/store/webhook_delivery_log.go` (new file)++```go+package store++import (+ "context"+ "database/sql"+)++// ResetPipelineNotification deletes the notification log for a plan.+// Called when user clicks BatchRunTasks to enable new notifications on retry.+func (s *Store) ResetPipelineNotification(ctx context.Context, planID int64) error {+ query := `DELETE FROM webhook_delivery_log WHERE plan_id = $1`+ _, err := s.db.ExecContext(ctx, query, planID)+ return err+}++// ClaimPipelineFailureNotification attempts to claim the right to send PIPELINE_FAILED webhook.+// Returns true if claimed (should send), false if already sent or claimed by another replica.+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.+func (s *Store) ClaimPipelineFailureNotification(ctx context.Context, planID int64) (bool, error) {+ query := `+ INSERT INTO webhook_delivery_log (plan_id, event_type)+ VALUES ($1, 'PIPELINE_FAILED')+ ON CONFLICT (plan_id) DO NOTHING+ RETURNING plan_id+ `++ var id int64+ err := s.db.QueryRowContext(ctx, query, planID).Scan(&id)+ if err == sql.ErrNoRows {+ return false, nil // Already exists+ }+ return err == nil, err+}++// ClaimPipelineCompletionNotification attempts to claim the right to send PIPELINE_COMPLETED webhook.+// Returns true if claimed (should send), false if already sent or claimed by another replica.+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.+func (s *Store) ClaimPipelineCompletionNotification(ctx context.Context, planID int64) (bool, error) {+ query := `+ INSERT INTO webhook_delivery_log (plan_id, event_type)+ VALUES ($1, 'PIPELINE_COMPLETED')+ ON CONFLICT (plan_id) DO NOTHING+ RETURNING plan_id+ `++ var id int64+ err := s.db.QueryRowContext(ctx, query, planID).Scan(&id)+ if err == sql.ErrNoRows {+ return false, nil // Already exists+ }+ return err == nil, err+}+```++## Example Flow++### Scenario 1: Initial Deployment with Failures++```+Initial deployment (all tasks attempt 0):+→ Task A fails+ → Replica 1: ClaimPipelineFailureNotification(plan=123)+ → INSERT succeeds → Send webhook: "Pipeline failed - Task A failed"++→ Task B fails+ → Replica 2: ClaimPipelineFailureNotification(plan=123)+ → ON CONFLICT (row already exists) → Skip, no duplicate webhook++Result: One webhook sent, no spam ✅+Database: (plan_id=123, event_type='PIPELINE_FAILED')+```++### Scenario 2: User Retries Failed Tasks++```+User clicks "Retry Tasks":+→ BatchRunTasks API called+ → DELETE FROM webhook_delivery_log WHERE plan_id=123+ → CreatePendingTaskRuns (creates attempt=1 for failed tasks)++Database: (empty - row deleted)++→ Task A succeeds (attempt=1)+→ Task B fails (attempt=1)+ → Replica 1: ClaimPipelineFailureNotification(plan=123)+ → INSERT succeeds (no conflict, row was deleted)+ → Send webhook: "Pipeline failed on retry - Task B failed"++Result: User gets feedback on retry ✅+Database: (plan_id=123, event_type='PIPELINE_FAILED')+```++### Scenario 3: Successful Completion++```+User retries again:+→ BatchRunTasks called+ → DELETE FROM webhook_delivery_log WHERE plan_id=123++→ All tasks succeed+ → checkPlanCompletion detects all tasks done, no failures+ → Replica 2: ClaimPipelineCompletionNotification(plan=123)+ → INSERT succeeds+ → Send webhook: "Pipeline completed successfully"++Result: Completion notification sent ✅+Database: (plan_id=123, event_type='PIPELINE_COMPLETED')+```++### Scenario 4: HA Race Condition Handling++```+Two replicas detect same task failure simultaneously:+→ Replica A: ClaimPipelineFailureNotification(plan=123)+ → INSERT INTO webhook_delivery_log VALUES (123, 'PIPELINE_FAILED')+ → Returns plan_id=123 → claimed=true → Sends webhook++→ Replica B: ClaimPipelineFailureNotification(plan=123) (100ms later)+ → INSERT INTO webhook_delivery_log VALUES (123, 'PIPELINE_FAILED')+ → ON CONFLICT (plan_id) DO NOTHING+ → Returns no rows → claimed=false → Skips webhook++Result: Only one webhook sent across HA replicas ✅+```++## Migration++Create migration file: `backend/migrator/migration/<<version>>/<<sequence>>##webhook_delivery_log.sql`++```sql+CREATE TABLE webhook_delivery_log (+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),+ event_type TEXT NOT NULL,+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()+);+```++## Cleanup++After implementation, remove in-memory channel:++1. **Remove from state:**+ ```go+ // backend/component/state/state.go+ // DELETE: TaskSkippedOrDoneChan chan int+ ```++2. **Remove senders:**+ ```go+ // backend/api/v1/rollout_service.go+ // DELETE: s.stateCfg.TaskSkippedOrDoneChan <- task.ID++ // backend/runner/taskrun/running_scheduler.go+ // DELETE: s.stateCfg.TaskSkippedOrDoneChan <- task.ID+ ```++3. **Remove receiver:**+ ```go+ // backend/runner/taskrun/scheduler.go+ // DELETE: case taskUID := <-s.stateCfg.TaskSkippedOrDoneChan:+ ```++## Benefits++✅ **HA-Compatible**: Database PRIMARY KEY prevents duplicate webhooks across replicas+✅ **No Spam**: One notification per failure/completion phase+✅ **Retry Support**: DELETE on BatchRunTasks resets state for fresh notifications+✅ **Immediate Delivery**: No polling or aggregation windows needed+✅ **Simple Schema**: Single table with PRIMARY KEY, no complex indexes+✅ **Audit Trail**: `event_type` shows which notification was sent+✅ **Follows Existing Pattern**: Same approach as TaskRun session monitoring (commit 564f1e5618)++## Comparison to Alternatives++### Alternative 1: 5-Minute Aggregation Windows+❌ Complex: Needs polling worker, window management+❌ Delayed: 5-minute wait before notification+❌ Race-prone: Multiple replicas might process same window++### Alternative 2: PostgreSQL Advisory Locks+❌ Lock held during webhook HTTP call (can timeout)+❌ More complex than simple INSERT/DELETE++### Alternative 3: In-Memory Channel (Current)+❌ Breaks in HA - not shared across replicas+❌ No persistence - lost on restart++### This Design (Database-Backed Log)+✅ Simple INSERT/DELETE operations+✅ Immediate notifications+✅ HA-safe with database constraints+✅ Stateless - any replica can handle requests++## Future Enhancements++- Add `retry_count` column to track how many times plan was retried+- Add `failed_task_count` to show aggregated metrics+- TTL cleanup for old delivery logs (e.g., delete after 30 days)++## References++- Similar pattern: TaskRun session monitoring HA design (commit 564f1e5618)+- Related: Webhook events redesign (docs/plans/2025-12-30-webhook-events-redesign.md)diff --git a/docs/plans/2025-12-31-ha-webhook-delivery-impl.md b/docs/plans/2025-12-31-ha-webhook-delivery-impl.mdnew file mode 100644index 0000000000..3727f9b3bf--- /dev/null+++ b/docs/plans/2025-12-31-ha-webhook-delivery-impl.md@@ -0,0 +1,738 @@+# HA-Compatible Webhook Delivery Implementation Plan++> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.++**Goal:** Implement database-backed webhook delivery to make PIPELINE_FAILED and PIPELINE_COMPLETED notifications work correctly in HA deployments++**Architecture:** Replace in-memory `TaskSkippedOrDoneChan` channel with `plan_webhook_delivery` table using PRIMARY KEY constraint for atomic claiming across replicas++**Tech Stack:** Go, PostgreSQL, existing webhook infrastructure++---++## Phase 1: Database Schema++### Task 1: Create Migration for plan_webhook_delivery Table++**Files:**+- Create: `backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql`++**Step 1: Create migration file**++Create `backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql`:++```sql+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).+-- One row per plan at any time - mutually exclusive events.+-- Row is deleted when user clicks BatchRunTasks to reset notification state.+CREATE TABLE plan_webhook_delivery (+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'+ event_type TEXT NOT NULL,+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()+);+```++**Step 2: Update LATEST.sql**++Add to `backend/migrator/migration/LATEST.sql` after the `plan_check_run` table (around line 231):++```sql+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).+-- One row per plan at any time - mutually exclusive events.+-- Row is deleted when user clicks BatchRunTasks to reset notification state.+CREATE TABLE plan_webhook_delivery (+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'+ event_type TEXT NOT NULL,+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()+);++ALTER SEQUENCE plan_webhook_delivery_plan_id_seq RESTART WITH 101;+```++**Step 3: Verify migration files**++Run: `ls backend/migrator/migration/3.14/`+Expected: See `0016##plan_webhook_delivery.sql`++**Step 4: Commit migration**++```bash+git add backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql backend/migrator/migration/LATEST.sql+git commit -m "feat: add plan_webhook_delivery table for HA webhook deduplication++Table provides atomic claim mechanism for webhook delivery across HA replicas+using PRIMARY KEY constraint. Deleted on BatchRunTasks to reset state.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++## Phase 2: Store Methods++### Task 2: Implement Store Methods for Webhook Delivery++**Files:**+- Create: `backend/store/plan_webhook_delivery.go`++**Step 1: Create store methods file**++Create `backend/store/plan_webhook_delivery.go`:++```go+package store++import (+ "context"+ "database/sql"+)++// ResetPlanWebhookDelivery deletes the webhook delivery record for a plan.+// Called when user clicks BatchRunTasks to enable new notifications on retry.+func (s *Store) ResetPlanWebhookDelivery(ctx context.Context, planID int64) error {+ query := `DELETE FROM plan_webhook_delivery WHERE plan_id = $1`+ _, err := s.db.ExecContext(ctx, query, planID)+ return err+}++// ClaimPipelineFailureNotification attempts to claim the right to send PIPELINE_FAILED webhook.+// Returns true if claimed (should send), false if already sent or claimed by another replica.+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.+func (s *Store) ClaimPipelineFailureNotification(ctx context.Context, planID int64) (bool, error) {+ query := `+ INSERT INTO plan_webhook_delivery (plan_id, event_type)+ VALUES ($1, 'PIPELINE_FAILED')+ ON CONFLICT (plan_id) DO NOTHING+ RETURNING plan_id+ `++ var id int64+ err := s.db.QueryRowContext(ctx, query, planID).Scan(&id)+ if err == sql.ErrNoRows {+ return false, nil // Already exists+ }+ return err == nil, err+}++// ClaimPipelineCompletionNotification attempts to claim the right to send PIPELINE_COMPLETED webhook.+// Returns true if claimed (should send), false if already sent or claimed by another replica.+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.+func (s *Store) ClaimPipelineCompletionNotification(ctx context.Context, planID int64) (bool, error) {+ query := `+ INSERT INTO plan_webhook_delivery (plan_id, event_type)+ VALUES ($1, 'PIPELINE_COMPLETED')+ ON CONFLICT (plan_id) DO NOTHING+ RETURNING plan_id+ `++ var id int64+ err := s.db.QueryRowContext(ctx, query, planID).Scan(&id)+ if err == sql.ErrNoRows {+ return false, nil // Already exists+ }+ return err == nil, err+}+```++**Step 2: Build to verify**++Run: `go build ./backend/store/...`+Expected: Build succeeds++**Step 3: Commit store methods**++```bash+git add backend/store/plan_webhook_delivery.go+git commit -m "feat: add store methods for plan webhook delivery++Implement atomic claim methods for PIPELINE_FAILED and PIPELINE_COMPLETED+webhooks using INSERT...ON CONFLICT for HA safety. Add reset method for+BatchRunTasks.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++## Phase 3: Integrate into BatchRunTasks++### Task 3: Reset Webhook State on BatchRunTasks++**Files:**+- Modify: `backend/api/v1/rollout_service.go:627-752`++**Step 1: Add reset call to BatchRunTasks**++In `backend/api/v1/rollout_service.go`, add after getting the plan (around line 655):++```go+// Reset notification state so user gets fresh feedback on retry+if err := s.store.ResetPlanWebhookDelivery(ctx, planID); err != nil {+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))+ // Don't fail the request - notification is non-critical+}+```++Full context - the function should look like:++```go+func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request[v1pb.BatchRunTasksRequest]) (*connect.Response[v1pb.BatchRunTasksResponse], error) {+ request := req.Msg+ // ... existing validation code ...++ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{+ ProjectID: &projectID,+ UID: &planID,+ })+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to find plan for rollout"))+ }+ if plan == nil {+ return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("rollout (plan) %v not found", planID))+ }++ // Reset notification state so user gets fresh feedback on retry+ if err := s.store.ResetPlanWebhookDelivery(ctx, planID); err != nil {+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))+ // Don't fail the request - notification is non-critical+ }++ // ... rest of existing code ...+}+```++**Step 2: Build to verify**++Run: `go build ./backend/api/v1/...`+Expected: Build succeeds++**Step 3: Commit BatchRunTasks integration**++```bash+git add backend/api/v1/rollout_service.go+git commit -m "feat: reset webhook delivery state on BatchRunTasks++Delete plan_webhook_delivery row when user retries tasks to enable fresh+notifications on retry attempts.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++## Phase 4: Send PIPELINE_FAILED on Task Failure++### Task 4: Add Helper to Get Failed Task Runs++**Files:**+- Modify: `backend/runner/taskrun/scheduler.go`++**Step 1: Add helper method to get failed task runs**++Add near the end of `backend/runner/taskrun/scheduler.go`:++```go+// getFailedTaskRuns returns all failed task runs for a plan to include in webhook payload.+func (s *Scheduler) getFailedTaskRuns(ctx context.Context, planID int64) []webhook.FailedTask {+ tasks, err := s.store.ListTasks(ctx, &store.TaskFind{PlanID: &planID})+ if err != nil {+ slog.Error("failed to list tasks for failed webhook", log.BBError(err))+ return nil+ }++ var failures []webhook.FailedTask+ for _, task := range tasks {+ if task.LatestTaskRunStatus != storepb.TaskRun_FAILED {+ continue+ }++ // Get the latest failed task run+ taskRuns, err := s.store.ListTaskRuns(ctx, &store.TaskRunFind{TaskID: &task.ID})+ if err != nil {+ slog.Error("failed to list task runs", log.BBError(err))+ continue+ }++ // Find the latest failed run+ var latestFailed *store.TaskRunMessage+ for _, tr := range taskRuns {+ if tr.Status == storepb.TaskRun_FAILED {+ if latestFailed == nil || tr.UpdatedAt.After(*latestFailed.UpdatedAt) {+ latestFailed = tr+ }+ }+ }++ if latestFailed == nil {+ continue+ }++ errorMsg := ""+ if latestFailed.Result != nil && latestFailed.Result.Error != "" {+ errorMsg = latestFailed.Result.Error+ }++ failures = append(failures, webhook.FailedTask{+ TaskID: int64(task.ID),+ TaskName: task.Name,+ DatabaseName: task.DatabaseName,+ InstanceName: task.Instance.ResourceID,+ ErrorMessage: errorMsg,+ FailedAt: *latestFailed.UpdatedAt,+ })+ }++ return failures+}+```++**Step 2: Build to verify**++Run: `go build ./backend/runner/taskrun/...`+Expected: Build succeeds++**Step 3: Commit helper method**++```bash+git add backend/runner/taskrun/scheduler.go+git commit -m "feat: add helper to get failed task runs for webhook++Collects all failed tasks in a plan with error messages for webhook payload.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++### Task 5: Send PIPELINE_FAILED Webhook on Task Failure++**Files:**+- Modify: `backend/runner/taskrun/running_scheduler.go`++**Step 1: Check where task is marked as FAILED**++Read `backend/runner/taskrun/running_scheduler.go` around line 200-240 to find where task status is set to FAILED.++Expected: Find code like:+```go+if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{+ ID: taskRun.ID,+ Status: storepb.TaskRun_FAILED,+ ...+}); err != nil {+ ...+}+```++**Step 2: Add webhook notification after marking task as FAILED**++After the task is marked as FAILED and before the return statement, add:++```go+// Send PIPELINE_FAILED webhook (HA-safe atomic claim)+s.sendPipelineFailureWebhook(ctx, task)+```++**Step 3: Implement sendPipelineFailureWebhook method**++Add to `backend/runner/taskrun/running_scheduler.go`:++```go+// sendPipelineFailureWebhook attempts to send PIPELINE_FAILED webhook.+// Uses atomic claim to prevent duplicate sends in HA deployments.+func (s *Scheduler) sendPipelineFailureWebhook(ctx context.Context, task *store.TaskMessage) {+ // Try to claim notification (only one replica succeeds)+ claimed, err := s.store.ClaimPipelineFailureNotification(ctx, task.PlanID)+ if err != nil {+ slog.Error("failed to claim pipeline failure notification", log.BBError(err))+ return+ }+ if !claimed {+ // Already sent by this or another replica+ return+ }++ // Get plan context+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})+ if err != nil || plan == nil {+ slog.Error("failed to get plan for failure webhook", log.BBError(err))+ return+ }++ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})+ if err != nil || project == nil {+ slog.Error("failed to get project for failure webhook", log.BBError(err))+ return+ }++ // Get all failed tasks+ failures := s.getFailedTaskRuns(ctx, task.PlanID)+ if len(failures) == 0 {+ slog.Warn("no failed tasks found for pipeline failure webhook", slog.Int64("plan_id", task.PlanID))+ return+ }++ // Send webhook+ s.webhookManager.CreateEvent(ctx, &webhook.Event{+ Type: storepb.Activity_PIPELINE_FAILED,+ Project: webhook.NewProject(project),+ PipelineFailed: &webhook.EventPipelineFailed{+ Rollout: webhook.NewRollout(plan),+ FailedTasks: failures,+ },+ })+}+```++**Step 4: Build to verify**++Run: `go build ./backend/runner/taskrun/...`+Expected: Build succeeds++**Step 5: Commit pipeline failure webhook**++```bash+git add backend/runner/taskrun/running_scheduler.go+git commit -m "feat: send PIPELINE_FAILED webhook on task failure++Add atomic claim-based webhook delivery when task fails. Only first failure+in a plan triggers notification, preventing spam in HA deployments.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++## Phase 5: Send PIPELINE_COMPLETED on Success++### Task 6: Update checkPlanCompletion to Send Webhook++**Files:**+- Modify: `backend/runner/taskrun/scheduler.go:112-210`++**Step 1: Add claim check before sending completion webhook**++In `backend/runner/taskrun/scheduler.go`, find the `checkPlanCompletion` function (around line 112).++After the `if hasFailures { return }` check (around line 167), add:++```go+// Try to claim completion notification (HA-safe)+claimed, err := s.store.ClaimPipelineCompletionNotification(ctx, planID)+if err != nil {+ slog.Error("failed to claim pipeline completion notification", log.BBError(err))+ return+}+if !claimed {+ return // Already sent+}+```++Full context - the section should look like:++```go+// Not all tasks complete yet+if !allComplete {+ return+}++// Always clear the failure window when plan completes+s.pipelineEvents.Clear(planID)++// Only send completion webhook if there were no failures+if hasFailures {+ return+}++// Try to claim completion notification (HA-safe)+claimed, err := s.store.ClaimPipelineCompletionNotification(ctx, planID)+if err != nil {+ slog.Error("failed to claim pipeline completion notification", log.BBError(err))+ return+}+if !claimed {+ return // Already sent+}++// ... existing webhook send code continues ...+```++**Step 2: Build to verify**++Run: `go build ./backend/runner/taskrun/...`+Expected: Build succeeds++**Step 3: Commit pipeline completion webhook**++```bash+git add backend/runner/taskrun/scheduler.go+git commit -m "feat: add atomic claim for PIPELINE_COMPLETED webhook++Prevent duplicate completion webhooks in HA deployments using database-backed+claim mechanism.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++## Phase 6: Cleanup In-Memory Channel++### Task 7: Remove TaskSkippedOrDoneChan Channel++**Files:**+- Modify: `backend/component/state/state.go`+- Modify: `backend/api/v1/rollout_service.go`+- Modify: `backend/runner/taskrun/running_scheduler.go`+- Modify: `backend/runner/taskrun/scheduler.go`++**Step 1: Remove channel from state**++In `backend/component/state/state.go`, remove these lines (around line 23-24):++```go+// DELETE these lines:+// TaskSkippedOrDoneChan is the channel for notifying the task is skipped or done.+TaskSkippedOrDoneChan chan int+```++And in the `New()` function (around line 37):++```go+// DELETE this line:+TaskSkippedOrDoneChan: make(chan int, 1000),+```++**Step 2: Remove sender in rollout_service.go**++In `backend/api/v1/rollout_service.go`, find and remove (around line 836):++```go+// DELETE these lines:+for _, task := range tasksToSkip {+ s.stateCfg.TaskSkippedOrDoneChan <- task.ID+}+```++**Step 3: Remove sender in running_scheduler.go**++In `backend/runner/taskrun/running_scheduler.go`, find and remove (around line 232):++```go+// DELETE this line:+s.stateCfg.TaskSkippedOrDoneChan <- task.ID+```++**Step 4: Remove receiver in scheduler.go**++In `backend/runner/taskrun/scheduler.go`, find the `runTaskCompletionListener` function (around line 82-110).++Remove the entire case statement:++```go+// DELETE this entire case block:+case taskUID := <-s.stateCfg.TaskSkippedOrDoneChan:+ if err := func() error {+ task, err := s.store.GetTaskByID(ctx, taskUID)+ if err != nil {+ return errors.Wrapf(err, "failed to get task")+ }++ // Check if entire plan is complete and handle webhooks+ s.checkPlanCompletion(ctx, task.PlanID)++ return nil+ }(); err != nil {+ slog.Error("failed to handle task completion", log.BBError(err))+ }+```++**Step 5: Build to verify**++Run: `go build ./backend/...`+Expected: Build succeeds++**Step 6: Commit channel removal**++```bash+git add backend/component/state/state.go backend/api/v1/rollout_service.go backend/runner/taskrun/running_scheduler.go backend/runner/taskrun/scheduler.go+git commit -m "refactor: remove TaskSkippedOrDoneChan in-memory channel++Remove in-memory channel replaced by database-backed webhook delivery.+TaskSkippedOrDoneChan is no longer needed as webhooks are triggered directly+in task failure and completion handlers.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++## Phase 7: Testing and Verification++### Task 8: Manual Testing++**Step 1: Start local Bytebase**++Run: `PG_URL=postgresql://bbdev@localhost/bbdev go run ./backend/bin/server/main.go --port 8080 --data . --debug`+Expected: Server starts, migration runs successfully++**Step 2: Verify table created**++Run: `psql -U bbdev bbdev -c "\d plan_webhook_delivery"`+Expected: See table schema with plan_id PRIMARY KEY++**Step 3: Test PIPELINE_FAILED webhook**++1. Create a plan with tasks+2. Make a task fail+3. Check webhook sent+4. Query: `SELECT * FROM plan_webhook_delivery;`+ Expected: One row with event_type='PIPELINE_FAILED'++**Step 4: Test retry doesn't send duplicate**++1. Fail another task in same plan+2. Check no duplicate webhook+3. Query: `SELECT COUNT(*) FROM plan_webhook_delivery WHERE plan_id = ?;`+ Expected: Still 1 row++**Step 5: Test BatchRunTasks resets state**++1. Click "Retry Tasks"+2. Query: `SELECT * FROM plan_webhook_delivery WHERE plan_id = ?;`+ Expected: 0 rows (deleted)+3. Fail task again+4. Check webhook sent again+5. Query: `SELECT COUNT(*) FROM plan_webhook_delivery WHERE plan_id = ?;`+ Expected: 1 row++**Step 6: Test PIPELINE_COMPLETED**++1. Retry and make all tasks succeed+2. Check PIPELINE_COMPLETED webhook sent+3. Query: `SELECT * FROM plan_webhook_delivery WHERE plan_id = ?;`+ Expected: One row with event_type='PIPELINE_COMPLETED'++**Step 7: Document test results**++Create test notes in commit message for final commit.++---++## Phase 8: Final Commit and Integration++### Task 9: Update Migrator Test++**Files:**+- Modify: `backend/migrator/migrator_test.go`++**Step 1: Find TestLatestVersion**++In `backend/migrator/migrator_test.go`, find `TestLatestVersion` function.++**Step 2: Update latest version**++Find the line that looks like:+```go+latestSchemaVersion := "3.14.15"+```++Increment the last number:+```go+latestSchemaVersion := "3.14.16"+```++**Step 3: Run test**++Run: `go test -v ./backend/migrator/ -run TestLatestVersion`+Expected: Test passes++**Step 4: Commit migrator test update**++```bash+git add backend/migrator/migrator_test.go+git commit -m "test: update latest schema version to 3.14.16++Update for plan_webhook_delivery table migration.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++### Task 10: Run Full Build and Lint++**Step 1: Format Go code**++Run: `gofmt -w backend/store/plan_webhook_delivery.go backend/api/v1/rollout_service.go backend/runner/taskrun/scheduler.go backend/runner/taskrun/running_scheduler.go backend/component/state/state.go`+Expected: Files formatted++**Step 2: Run golangci-lint**++Run: `golangci-lint run --allow-parallel-runners`+Expected: No errors (run multiple times until clean)++**Step 3: Auto-fix lint issues**++Run: `golangci-lint run --fix --allow-parallel-runners`+Expected: Auto-fixable issues resolved++**Step 4: Build backend**++Run: `go build -ldflags "-w -s" -p=16 -o ./bytebase-build/bytebase ./backend/bin/server/main.go`+Expected: Build succeeds++**Step 5: Commit any lint fixes**++```bash+git add -A+git commit -m "chore: fix linting issues++Auto-fix and manual lint issue resolution.++🤖 Generated with Claude Code++Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"+```++---++## Completion Checklist++- [ ] Migration created for `plan_webhook_delivery` table+- [ ] LATEST.sql updated with new table+- [ ] Store methods implemented (Claim, Reset)+- [ ] BatchRunTasks resets webhook state+- [ ] PIPELINE_FAILED webhook sent on task failure+- [ ] PIPELINE_COMPLETED webhook sent on plan completion+- [ ] In-memory `TaskSkippedOrDoneChan` removed+- [ ] Migrator test updated+- [ ] Code formatted and linted+- [ ] Manual testing completed+- [ ] All commits have proper messages++## Notes++- Each webhook is sent exactly once per "phase" (initial run, each retry)+- HA-safe: PRIMARY KEY constraint prevents duplicates across replicas+- User action (BatchRunTasks) explicitly resets state+- No polling or background workers needed+- Immediate notification delivery